Skip to content

Automate browser based workflows with AI

feat(SKY-9570): turn classify-UNKNOWN into a productive learning signal (#5882)

pedrohsdbc587bfb6f5ba
diff --git a/skyvern/core/script_generations/real_skyvern_page_ai.py b/skyvern/core/script_generations/real_skyvern_page_ai.py
index af157f3c1..4a49cb94d 100644
--- a/skyvern/core/script_generations/real_skyvern_page_ai.py
+++ b/skyvern/core/script_generations/real_skyvern_page_ai.py
@@ -594,6 +594,12 @@ class RealSkyvernPageAi(SkyvernPageAi):
"""
current_url = self.page.url
result = "UNKNOWN"
+ # Defaults so the meta write below is always safe — the LLM-exception
+ # path skips the assignments inside the try block, but UNKNOWN can
+ # still be returned and the metadata fields must be defined.
+ reasoning: str = ""
+ confidence: float = 0.0
+ extracted_text_for_meta: str = ""
# Tier 0: URL pattern matching (FREE)
if url_patterns:
@@ -608,7 +614,12 @@ class RealSkyvernPageAi(SkyvernPageAi):
)
result = key
await self._record_branch_hit(result)
- self._store_classify_result(result)
+ self._store_classify_result(
+ result,
+ current_url=current_url,
+ options=options,
+ block_label=self.current_label,
+ )
return result
except re.error:
LOG.warning("page.classify: invalid URL regex pattern", key=key, pattern=pattern)
@@ -633,18 +644,23 @@ class RealSkyvernPageAi(SkyvernPageAi):
)
result = key
await self._record_branch_hit(result)
- self._store_classify_result(result)
+ self._store_classify_result(
+ result,
+ current_url=current_url,
+ options=options,
+ block_label=self.current_label,
+ )
return result
# Tier 2: Mini-LLM classification
if not scraped:
await self._refresh_scraped_page(take_screenshots=False)
- extracted_text = (self.scraped_page.extracted_text or "")[:2000]
+ extracted_text_for_meta = (self.scraped_page.extracted_text or "")[:2000]
classify_prompt = prompt_engine.load_prompt(
template="page-classify",
current_url=current_url,
- extracted_text=extracted_text,
+ extracted_text=extracted_text_for_meta,
options=options,
)
@@ -684,7 +700,15 @@ class RealSkyvernPageAi(SkyvernPageAi):
result = "UNKNOWN"
await self._record_branch_hit(result)
- self._store_classify_result(result)
+ self._store_classify_result(
+ result,
+ current_url=current_url,
+ options=options,
+ block_label=self.current_label,
+ rejection_reasoning=reasoning,
+ confidence=confidence,
+ text_excerpt=extracted_text_for_meta,
+ )
return result
async def _record_element_fallback_episode(
@@ -794,14 +818,36 @@ class RealSkyvernPageAi(SkyvernPageAi):
LOG.warning("Failed to record element fallback episode for selector failure", exc_info=True)
@staticmethod
- def _store_classify_result(result: str) -> None:
- """Store the classify result on SkyvernContext for fallback episode recording."""
+ def _store_classify_result(
+ result: str,
+ *,
+ current_url: str,
+ options: dict[str, str],
+ block_label: str | None,
+ rejection_reasoning: str = "",
+ confidence: float = 0.0,
+ text_excerpt: str = "",
+ ) -> None:
+ """Store classify result + (on UNKNOWN) rejection metadata for the next element_fallback to consume."""
try:
ctx = skyvern_context.current()
- if ctx:
- ctx.last_classify_result = result
+ if ctx is None:
+ return
+ ctx.last_classify_result = result
+ if result == "UNKNOWN":
+ ctx.last_classify_meta = {
+ "result": "UNKNOWN",
+ "url_at_classify": current_url,
+ "block_label_at_classify": block_label,
+ "candidate_options": dict(options),
+ "rejection_reasoning": rejection_reasoning,
+ "confidence": confidence,
+ "text_excerpt": (text_excerpt or "")[:500],
+ }
+ else:
+ ctx.last_classify_meta = None
except Exception:
- pass
+ LOG.debug("_store_classify_result: failed to persist classify meta", exc_info=True)
async def _record_branch_hit(self, branch_key: str) -> None:
"""Best-effort recording of a classify branch hit for TTL tracking."""
@@ -824,16 +870,20 @@ class RealSkyvernPageAi(SkyvernPageAi):
navigation_goal: str,
max_steps: int = 5,
) -> None:
- """Activate the AI agent from the CURRENT page position to achieve a navigation goal.
+ """Drive an AI agent from the current page state toward ``navigation_goal``.
- Uses repeated ai_act calls with ai_validate checks to execute from the
- current page state. Records all actions taken for later review by the
- AI Script Reviewer.
+ Records one fallback episode per invocation regardless of exit path; pops
+ any preceding classify metadata at entry so subsequent fallbacks cannot
+ inherit it.
"""
context = skyvern_context.current()
if not context or not context.organization_id:
raise Exception("element_fallback requires an active context with organization_id")
+ # Consume-at-entry — exception-safe pop.
+ classify_meta: dict[str, Any] | None = context.last_classify_meta
+ context.last_classify_meta = None
+
LOG.info(
"page.element_fallback: starting from current page",
navigation_goal=navigation_goal,
@@ -842,7 +892,6 @@ class RealSkyvernPageAi(SkyvernPageAi):
workflow_run_id=context.workflow_run_id,
)
- # Capture page state before element fallback for episode recording
page_url_at_entry = self.page.url
page_text_at_entry: str | None = None
try:
@@ -852,55 +901,63 @@ class RealSkyvernPageAi(SkyvernPageAi):
steps_taken: list[dict] = []
completed = False
+ exception_summary: str | None = None
+ captured_exception: BaseException | None = None
+
+ try:
+ for step_num in range(max_steps):
+ is_complete = await self.ai_validate(
+ prompt=f"Has the following goal been achieved? Goal: {navigation_goal}",
+ )
+ if is_complete:
+ LOG.info(
+ "page.element_fallback: goal achieved",
+ step_num=step_num,
+ navigation_goal=navigation_goal,
+ )
+ completed = True
+ break
- for step_num in range(max_steps):
- # Check if the goal has been achieved
- is_complete = await self.ai_validate(
- prompt=f"Has the following goal been achieved? Goal: {navigation_goal}",
- )
- if is_complete:
LOG.info(
- "page.element_fallback: goal achieved",
+ "page.element_fallback: executing step",
step_num=step_num,
- navigation_goal=navigation_goal,
+ current_url=self.page.url,
)
- completed = True
- break
-
- LOG.info(
- "page.element_fallback: executing step",
- step_num=step_num,
- current_url=self.page.url,
- )
- # Let the AI agent take an action toward the goal
- await self.ai_act(
- prompt=f"Take the next action to achieve this goal: {navigation_goal}",
+ context.current_step_actions = []
+ url_before = self.page.url
+ try:
+ await self.ai_act(
+ prompt=f"Take the next action to achieve this goal: {navigation_goal}",
+ )
+ actions_this_step = list(context.current_step_actions or [])
+ finally:
+ context.current_step_actions = None
+ steps_taken.append(
+ {
+ "step": step_num,
+ "url_before": url_before,
+ "url_after": self.page.url,
+ "actions": actions_this_step[:10],
+ }
+ )
+ except Exception as exc:
+ exception_summary = f"{type(exc).__name__}: {exc!s}"[:500]
+ captured_exception = exc
+ finally:
+ await self._persist_classify_unknown_episode(
+ context=context,
+ classify_meta=classify_meta,
+ page_url_at_entry=page_url_at_entry,
+ page_text_at_entry=page_text_at_entry,
+ steps_taken=steps_taken,
+ navigation_goal=navigation_goal,
+ completed=completed,
+ exception_summary=exception_summary,
)
- steps_taken.append({"step": step_num, "url_after": self.page.url})
- # Record an element fallback episode for the feedback loop
- if context.workflow_run_id and context.workflow_permanent_id:
- try:
- await app.DATABASE.scripts.create_fallback_episode(
- organization_id=context.organization_id,
- workflow_permanent_id=context.workflow_permanent_id,
- workflow_run_id=context.workflow_run_id,
- block_label=self.current_label or "unknown",
- fallback_type="element",
- script_revision_id=context.script_revision_id,
- error_message=f"classify returned UNKNOWN, element_fallback goal: {navigation_goal}",
- page_url=page_url_at_entry,
- page_text_snapshot=page_text_at_entry,
- agent_actions={
- "navigation_goal": navigation_goal,
- "completed": completed,
- "steps_taken": len(steps_taken),
- "steps": steps_taken[:20],
- },
- )
- except Exception:
- LOG.debug("Failed to record element fallback episode", exc_info=True)
+ if captured_exception is not None:
+ raise captured_exception
if not completed:
LOG.warning(
@@ -910,6 +967,80 @@ class RealSkyvernPageAi(SkyvernPageAi):
)
raise Exception(f"Element fallback did not complete within {max_steps} steps")
+ async def _persist_classify_unknown_episode(
+ self,
+ *,
+ context: skyvern_context.SkyvernContext,
+ classify_meta: dict[str, Any] | None,
+ page_url_at_entry: str,
+ page_text_at_entry: str | None,
+ steps_taken: list[dict],
+ navigation_goal: str,
+ completed: bool,
+ exception_summary: str | None,
+ ) -> None:
+ """Record the fallback episode; attach classify_meta only on (URL, block_label) match."""
+ if not (context.organization_id and context.workflow_run_id and context.workflow_permanent_id):
+ LOG.debug("Skipping fallback episode: missing context identifiers")
+ return
+
+ # Single narrow: ``meta`` is the trustworthy classify metadata or None.
+ # Carries the URL+block_label predecessor invariant.
+ meta: dict[str, Any] | None = None
+ if (
+ classify_meta is not None
+ and classify_meta.get("url_at_classify") == page_url_at_entry
+ and classify_meta.get("block_label_at_classify") == self.current_label
+ ):
+ meta = classify_meta
+
+ classify_result_for_episode: str | None = meta.get("result") if meta is not None else None
+
+ # Flat ``actions`` matches the reviewer template iteration; ``steps`` keeps the per-iteration grouping.
+ flat_actions: list[dict[str, Any]] = []
+ for step in steps_taken:
+ for act in step.get("actions") or []:
+ flat_actions.append(act)
+ agent_actions_payload: dict[str, Any] = {
+ "navigation_goal": navigation_goal,
+ "completed": completed,
+ "steps_taken": len(steps_taken),
+ "actions": flat_actions[:20],
+ "steps": steps_taken[:20],
+ }
+ if exception_summary is not None:
+ agent_actions_payload["exception_summary"] = exception_summary
+ if meta is not None and meta.get("result") == "UNKNOWN":
+ agent_actions_payload["classify_rejection"] = {
+ "reasoning": meta.get("rejection_reasoning"),
+ "confidence": meta.get("confidence"),
+ "candidate_options": meta.get("candidate_options"),
+ "text_excerpt": meta.get("text_excerpt"),
+ }
+
+ error_message = (
+ f"classify returned UNKNOWN, element_fallback goal: {navigation_goal}"
+ if classify_result_for_episode == "UNKNOWN"
+ else f"element_fallback goal: {navigation_goal}"
+ )
+
+ try:
+ await app.DATABASE.scripts.create_fallback_episode(
+ organization_id=context.organization_id,
+ workflow_permanent_id=context.workflow_permanent_id,
+ workflow_run_id=context.workflow_run_id,
+ block_label=self.current_label or "unknown",
+ fallback_type="element",
+ script_revision_id=context.script_revision_id,
+ error_message=error_message,
+ classify_result=classify_result_for_episode,
+ page_url=page_url_at_entry,
+ page_text_snapshot=page_text_at_entry,
+ agent_actions=agent_actions_payload,
+ )
+ except Exception:
+ LOG.debug("Failed to record element fallback episode", exc_info=True)
+
async def ai_extract(
self,
prompt: str,
@@ -1387,6 +1518,58 @@ class RealSkyvernPageAi(SkyvernPageAi):
)
return
+ # ``current_step_actions`` is None outside element_fallback frames; field names match the reviewer template.
+ if context.current_step_actions is not None:
+ # ai_act treats ``result is None or empty`` as success (no
+ # exception raised below). Label "failed" only when there's
+ # positive evidence of failure — otherwise "success" — to
+ # match the existing ``if result and result[-1].success is
+ # False: raise`` semantics two lines below.
+ action_failed = bool(result) and result[-1].success is False
+ action_record: dict[str, Any] = {
+ "action_type": action_type,
+ "intention": getattr(action, "intention", None),
+ "reasoning": getattr(action, "reasoning", None),
+ "page_url": self.page.url,
+ "status": "failed" if action_failed else "success",
+ }
+ el_data = getattr(action, "skyvern_element_data", None)
+ if el_data:
+ action_record["element_text"] = (el_data.get("text") or "")[:200]
+ action_record["element_tag"] = el_data.get("tagName", "")
+ sel_options = compute_selector_options(el_data)
+ if sel_options:
+ action_record["selector_options"] = sel_options
+ action_record["css_suggestion"] = sel_options[0][0]
+ el_attrs = el_data.get("attributes") or {}
+ if el_attrs:
+ action_record["all_attributes"] = {
+ k: v
+ for k, v in el_attrs.items()
+ if k
+ in (
+ "id",
+ "name",
+ "class",
+ "aria-label",
+ "placeholder",
+ "type",
+ "role",
+ "data-testid",
+ "href",
+ "value",
+ "title",
+ )
+ }
+ if hasattr(action, "get_xpath"):
+ try:
+ xpath = action.get_xpath()
+ except Exception:
+ xpath = None
+ if xpath:
+ action_record["xpath"] = xpath
+ context.current_step_actions.append(action_record)
+
if result and result[-1].success is False:
raise Exception(result[-1].exception_message)
diff --git a/skyvern/forge/prompts/skyvern/script-reviewer-extraction.j2 b/skyvern/forge/prompts/skyvern/script-reviewer-extraction.j2
index 4dce563ef..5dca98631 100644
--- a/skyvern/forge/prompts/skyvern/script-reviewer-extraction.j2
+++ b/skyvern/forge/prompts/skyvern/script-reviewer-extraction.j2
@@ -86,6 +86,22 @@ The cached code failed and the AI agent took over. Here is what happened:
- **Fallback Type**: {{ episode.fallback_type }}
- **Error**: {{ episode.error_message or "N/A" }}
- **Classify Result**: {{ episode.classify_result or "N/A" }}
+{% if episode.classify_result == "UNKNOWN" %}
+ ⚠️ `page.classify()` returned UNKNOWN — the runtime LLM rejected every named option. Add a new branch with URL/text patterns derived from the page data below.
+{% endif %}
+{% if episode.agent_actions is mapping and episode.agent_actions.classify_rejection is defined %}
+- **Why the runtime LLM rejected the options**:
+ > {{ episode.agent_actions.classify_rejection.reasoning or "(no reasoning recorded)" }}
+{% if episode.agent_actions.classify_rejection.candidate_options %}
+ Options the LLM was given:
+{%- for opt_key, opt_desc in episode.agent_actions.classify_rejection.candidate_options.items() %}
+ - `{{ opt_key }}`: {{ opt_desc }}
+{%- endfor %}
+{% endif %}
+{% endif %}
+{% if episode.agent_actions is mapping and episode.agent_actions.exception_summary is defined and episode.agent_actions.exception_summary %}
+- **Recovery agent crashed mid-fallback**: `{{ episode.agent_actions.exception_summary }}`
+{% endif %}
- **Page URL at failure**: {{ episode.page_url or "N/A" }}
- **Page Text at failure** (use this to derive text_patterns):
```
diff --git a/skyvern/forge/prompts/skyvern/script-reviewer-form-filling.j2 b/skyvern/forge/prompts/skyvern/script-reviewer-form-filling.j2
index 5acf33b46..16fee76e9 100644
--- a/skyvern/forge/prompts/skyvern/script-reviewer-form-filling.j2
+++ b/skyvern/forge/prompts/skyvern/script-reviewer-form-filling.j2
@@ -91,6 +91,22 @@ The cached code failed and the AI agent took over. Here is what happened:
- **Block Label**: {{ episode.block_label }}
- **Fallback Type**: {{ episode.fallback_type }}
- **Error**: {{ episode.error_message or "N/A" }}
+{% if episode.classify_result == "UNKNOWN" %}
+ ⚠️ `page.classify()` returned UNKNOWN — the runtime LLM rejected every named option. Add a new branch (prefer URL/text patterns over LLM-only options).
+{% endif %}
+{% if episode.agent_actions is mapping and episode.agent_actions.classify_rejection is defined %}
+- **Why the runtime LLM rejected the options**:
+ > {{ episode.agent_actions.classify_rejection.reasoning or "(no reasoning recorded)" }}
+{% if episode.agent_actions.classify_rejection.candidate_options %}
+ Options the LLM was given:
+{%- for opt_key, opt_desc in episode.agent_actions.classify_rejection.candidate_options.items() %}
+ - `{{ opt_key }}`: {{ opt_desc }}
+{%- endfor %}
+{% endif %}
+{% endif %}
+{% if episode.agent_actions is mapping and episode.agent_actions.exception_summary is defined and episode.agent_actions.exception_summary %}
+- **Recovery agent crashed mid-fallback**: `{{ episode.agent_actions.exception_summary }}`
+{% endif %}
- **Page URL at failure**: {{ episode.page_url or "N/A" }}
- **Page Text at failure**:
```
diff --git a/skyvern/forge/prompts/skyvern/script-reviewer.j2 b/skyvern/forge/prompts/skyvern/script-reviewer.j2
index d339cc918..4488160bc 100644
--- a/skyvern/forge/prompts/skyvern/script-reviewer.j2
+++ b/skyvern/forge/prompts/skyvern/script-reviewer.j2
@@ -122,9 +122,27 @@ The cached code failed and the AI agent took over. Here is what happened:
- **Fallback Type**: {{ episode.fallback_type }}
- **Error**: {{ episode.error_message or "N/A" }}
- **Classify Result**: {{ episode.classify_result or "N/A" }}
-{% if episode.classify_result and episode.classify_result != "N/A" %}
+{% if episode.classify_result == "UNKNOWN" %}
+ ⚠️ `page.classify()` returned UNKNOWN — the runtime LLM rejected every named option. This is the signal to ADD a new option/branch covering this page state. Use the rejection reasoning and candidate options below as the seed for the new option's description, and prefer URL/text patterns derived from the **Page URL at failure** and **Page Text at failure** fields below for deterministic dispatch.
+{% elif episode.classify_result and episode.classify_result != "N/A" %}
⚠️ The classify matched `{{ episode.classify_result }}` but its branch actions STILL FAILED. Update selectors for this branch using the episode data below.
{% endif %}
+{% if episode.agent_actions is mapping and episode.agent_actions.classify_rejection is defined %}
+- **Why the runtime LLM rejected the options** (verbatim mini-LLM reasoning at classify time):
+ > {{ episode.agent_actions.classify_rejection.reasoning or "(no reasoning recorded)" }}
+{% if episode.agent_actions.classify_rejection.confidence is defined %}
+ Confidence in rejection: {{ "%.2f"|format(episode.agent_actions.classify_rejection.confidence) }}.
+{% endif %}
+{% if episode.agent_actions.classify_rejection.candidate_options %}
+ Options the LLM was given (and rejected):
+{%- for opt_key, opt_desc in episode.agent_actions.classify_rejection.candidate_options.items() %}
+ - `{{ opt_key }}`: {{ opt_desc }}
+{%- endfor %}
+{% endif %}
+{% endif %}
+{% if episode.agent_actions is mapping and episode.agent_actions.exception_summary is defined and episode.agent_actions.exception_summary %}
+- **Recovery agent crashed mid-fallback**: `{{ episode.agent_actions.exception_summary }}` — treat the partial action trace below as a best-effort hint, not a complete recovery recipe.
+{% endif %}
- **Page URL at failure**: {{ episode.page_url or "N/A" }}
- **Page Text at failure** (use this to derive text_patterns):
```
@@ -250,7 +268,15 @@ If the existing code already has a `page.wait()` and it still failed, increase t
## What to do
{% for episode in episodes %}
-{% if episode.classify_result and episode.classify_result != "N/A" and episode.classify_result != "UNKNOWN" %}
+{% if episode.classify_result == "UNKNOWN" %}
+### Adding a new branch for this novel page state
+The classify returned UNKNOWN — its candidate options (shown above) didn't fit the page. Add a new option that DOES fit:
+1. **Prefer adding** a new `elif` branch with a `text_patterns` or `url_patterns` entry over rewording an existing option's description. The mini-LLM tier is brittle; URL/text matching is deterministic.
+2. **Seed the new option's description** from the rejection reasoning shown above ("Why the runtime LLM rejected the options"). It already explains what page state is actually present.
+3. **Derive the pattern** from the recorded **Page URL at failure** (URL fragments unique to this state) or **Page Text at failure** (a substring that uniquely identifies the page). Add the pattern to the `text_patterns=` or `url_patterns=` argument of the existing `page.classify()` call AND add the matching option to `options=`.
+4. **Translate the recorded agent actions** into the new branch body, using selector/element data per the rules below.
+5. **Do NOT remove the `else` branch.** It still needs to call `await page.element_fallback(navigation_goal="...")` to catch any genuinely-new states the new branch doesn't cover.
+{% elif episode.classify_result and episode.classify_result != "N/A" %}
### Handling episode with existing branch `{{ episode.classify_result }}`
The classify routed to `{{ episode.classify_result }}` but its actions failed. Update the branch's selectors:
1. Keep existing selectors — ADD new label variants with comma-separated CSS selectors.
diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.py
index d152baa8c..c8a4c8387 100644
--- a/skyvern/forge/sdk/core/skyvern_context.py
+++ b/skyvern/forge/sdk/core/skyvern_context.py
@@ -90,6 +90,8 @@ class SkyvernContext:
ai_mode_override: str | None = None
script_llm_call_count: int = 0
last_classify_result: str | None = None
+ last_classify_meta: dict[str, Any] | None = None
+ current_step_actions: list[dict[str, Any]] | None = None
skip_complete_verification: bool = False
# magic link handling
diff --git a/skyvern/services/script_reviewer.py b/skyvern/services/script_reviewer.py
index ebcc85e4e..2fd82a7ac 100644
--- a/skyvern/services/script_reviewer.py
+++ b/skyvern/services/script_reviewer.py
@@ -68,6 +68,10 @@ async def load_filtered_run_param_values(workflow_run_id: str) -> dict[str, str]
return result
+class AutoFixMissingElseGoalError(ValueError):
+ """Raised when an auto-injected else-branch element_fallback would need a navigation_goal but none exists."""
+
+
@dataclasses.dataclass(frozen=True)
class BlockReviewResult:
"""Result of reviewing a single block, carrying the code and LLM artifacts."""
@@ -1090,9 +1094,9 @@ class ScriptReviewer:
marker_ids=[c.marker_id for c in recoverable_candidates[:10]],
)
- # Use provided navigation goal, or fall back to a generic description
- if not navigation_goal:
- navigation_goal = "Complete the navigation task for this block"
+ # Empty string is safe for downstream re.findall / prompt rendering;
+ # ``_auto_fix_missing_else`` raises if a real goal is actually required.
+ navigation_goal_for_prompt = navigation_goal or ""
# Infer function signature from existing code
function_signature = self._extract_function_signature(existing_code)
@@ -1127,7 +1131,7 @@ class ScriptReviewer:
# Extract parameter keys from {{ param }} placeholders in the navigation goal
# and merge with workflow-level parameter keys for a complete list.
- goal_param_keys = set(re.findall(r"\{\{\s*(\w+)\s*\}\}", navigation_goal))
+ goal_param_keys = set(re.findall(r"\{\{\s*(\w+)\s*\}\}", navigation_goal_for_prompt))
parameter_keys = sorted(goal_param_keys | set(all_parameter_keys or []))
# The filter strips `<block_label>_output` from the valid set so the
# validator's primary mode rejects any ref to the current block's own
@@ -1165,7 +1169,7 @@ class ScriptReviewer:
# Build the reviewer prompt
reviewer_prompt = prompt_engine.load_prompt(
template=template,
- navigation_goal=navigation_goal,
+ navigation_goal=navigation_goal_for_prompt,
existing_code=existing_code,
episodes=[
{
@@ -1326,8 +1330,18 @@ class ScriptReviewer:
# Validate classify handling (every classify must have an else branch)
classify_error = self._validate_classify_handling(updated_code)
if classify_error is not None:
- # Try auto-fixing by injecting missing else branches
- fixed_code = self._auto_fix_missing_else(updated_code, navigation_goal or "")
+ # Try auto-fixing by injecting missing else branches.
+ fixed_code: str | None
+ try:
+ fixed_code = self._auto_fix_missing_else(updated_code, navigation_goal)
+ except AutoFixMissingElseGoalError as exc:
+ LOG.warning(
+ "ScriptReviewer: skipped auto-fix-else for goal-less block",
+ block_label=block_label,
+ script_revision_id=script_revision_id,
+ reason=str(exc),
+ )
+ fixed_code = None
revalidation_error = (
self._validate_classify_handling(fixed_code) if fixed_code else "auto-fix returned None"
)
@@ -2917,12 +2931,13 @@ class ScriptReviewer:
+ " Each return statement must use a next_block_label and branch_index from the branch definitions."
)
- def _auto_fix_missing_else(self, code: str, navigation_goal: str) -> str | None:
- """Auto-inject missing `else` branches after page.classify() if/elif chains.
+ def _auto_fix_missing_else(self, code: str, navigation_goal: str | None) -> str | None:
+ """Auto-inject missing ``else`` branches after page.classify() if/elif chains.
- When the LLM generates a classify with if/elif but no else, this inserts
- an else branch with element_fallback. Returns the fixed code, or None
- if the code structure is too complex to safely auto-fix.
+ Returns the fixed code, or None when no injection is performed (no
+ classify calls present, or the structure is too complex to safely
+ auto-fix). Raises ``AutoFixMissingElseGoalError`` if injection is
+ needed but no ``navigation_goal`` is available.
"""
lines = code.split("\n")
classify_line_indices = []
@@ -2933,6 +2948,14 @@ class ScriptReviewer:
if not classify_line_indices:
return None
+ # Defer the goal check to injection sites — classify-already-has-else short-circuits without it.
+ def _require_goal() -> str:
+ if not navigation_goal or not navigation_goal.strip():
+ raise AutoFixMissingElseGoalError(
+ "Cannot auto-inject else-branch element_fallback: block has no navigation_goal."
+ )
+ return navigation_goal
+
# Process in reverse order so line insertions don't shift earlier indices
for classify_line_idx in reversed(classify_line_indices):
if_indent: int | None = None
@@ -2973,8 +2996,9 @@ class ScriptReviewer:
if last_branch_body_end is not None:
indent_str = " " * if_indent
body_indent = " " * (if_indent + 4)
+ _goal_str = _require_goal()
goal = (
- navigation_goal.replace("\\", "\\\\")
+ _goal_str.replace("\\", "\\\\")
.replace('"', '\\"')
.replace("\n", " ")
.replace("\r", " ")[:200]
@@ -2992,8 +3016,9 @@ class ScriptReviewer:
if if_indent is not None and last_branch_body_end is not None:
indent_str = " " * if_indent
body_indent = " " * (if_indent + 4)
+ _goal_str = _require_goal()
goal = (
- navigation_goal.replace("\\", "\\\\")
+ _goal_str.replace("\\", "\\\\")
.replace('"', '\\"')
.replace("\n", " ")
.replace("\r", " ")[:200]
@@ -3010,11 +3035,9 @@ class ScriptReviewer:
if if_indent is not None and last_branch_body_end is not None:
indent_str = " " * if_indent
body_indent = " " * (if_indent + 4)
+ _goal_str = _require_goal()
goal = (
- navigation_goal.replace("\\", "\\\\")
- .replace('"', '\\"')
- .replace("\n", " ")
- .replace("\r", " ")[:200]
+ _goal_str.replace("\\", "\\\\").replace('"', '\\"').replace("\n", " ").replace("\r", " ")[:200]
)
else_block = [
f"{indent_str}else:",
diff --git a/tests/unit/test_auto_fix_missing_else_goal.py b/tests/unit/test_auto_fix_missing_else_goal.py
new file mode 100644
index 000000000..87f66178f
--- /dev/null
+++ b/tests/unit/test_auto_fix_missing_else_goal.py
@@ -0,0 +1,74 @@
+"""Tests for ``_auto_fix_missing_else`` goal-required behavior."""
+
+from __future__ import annotations
+
+import pytest
+
+from skyvern.services.script_reviewer import AutoFixMissingElseGoalError, ScriptReviewer
+
+CLASSIFY_NO_ELSE_CODE = """
+async def block_fn(page, context):
+ state = await page.classify(options={"a": "page A"})
+ if state == "a":
+ await page.click(selector="#a-button")
+"""
+
+
+CLASSIFY_WITH_ELSE_CODE = """
+async def block_fn(page, context):
+ state = await page.classify(options={"a": "page A"})
+ if state == "a":
+ await page.click(selector="#a-button")
+ else:
+ await page.element_fallback(navigation_goal="Click the A button")
+"""
+
+
+NO_CLASSIFY_CODE = """
+async def block_fn(page, context):
+ await page.click(selector="#submit")
+"""
+
+
+class TestAutoFixMissingElseGoal:
+ def setup_method(self) -> None:
+ self.reviewer = ScriptReviewer()
+
+ def test_raises_on_missing_goal_when_else_needs_injection(self) -> None:
+ with pytest.raises(AutoFixMissingElseGoalError):
+ self.reviewer._auto_fix_missing_else(CLASSIFY_NO_ELSE_CODE, None)
+
+ def test_raises_on_empty_string_goal(self) -> None:
+ with pytest.raises(AutoFixMissingElseGoalError):
+ self.reviewer._auto_fix_missing_else(CLASSIFY_NO_ELSE_CODE, "")
+
+ def test_raises_on_whitespace_only_goal(self) -> None:
+ with pytest.raises(AutoFixMissingElseGoalError):
+ self.reviewer._auto_fix_missing_else(CLASSIFY_NO_ELSE_CODE, " \n\t ")
+
+ def test_no_classify_returns_none_without_raising(self) -> None:
+ # No classify in the code → nothing to inject → None, no goal needed.
+ assert self.reviewer._auto_fix_missing_else(NO_CLASSIFY_CODE, None) is None
+ assert self.reviewer._auto_fix_missing_else(NO_CLASSIFY_CODE, "") is None
+
+ def test_real_goal_threaded_into_injected_else(self) -> None:
+ fixed = self.reviewer._auto_fix_missing_else(
+ CLASSIFY_NO_ELSE_CODE,
+ "Click the A button on the search results page",
+ )
+ assert fixed is not None
+ assert "else:" in fixed
+ assert "page.element_fallback" in fixed
+ assert "Click the A button on the search results page" in fixed
+ # And the placeholder string must NOT appear.
+ assert "Complete the navigation task for this block" not in fixed
+
+ def test_existing_else_branch_is_untouched(self) -> None:
+ # Even with a missing goal — there's no injection needed, so no raise.
+ result = self.reviewer._auto_fix_missing_else(CLASSIFY_WITH_ELSE_CODE, None)
+ # _auto_fix_missing_else returns the (possibly-edited) code; if there
+ # was nothing to fix, the code is returned unchanged. Either None or
+ # the original-equivalent string is acceptable as long as no raise
+ # occurs.
+ if result is not None:
+ assert "Click the A button" in result
diff --git a/tests/unit/test_classify_metadata_lifecycle.py b/tests/unit/test_classify_metadata_lifecycle.py
new file mode 100644
index 000000000..afca60156
--- /dev/null
+++ b/tests/unit/test_classify_metadata_lifecycle.py
@@ -0,0 +1,175 @@
+"""Tests for ``_store_classify_result`` metadata lifecycle on SkyvernContext
+and the consume-at-entry invariant in ``ai_element_fallback``.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from skyvern.core.script_generations.real_skyvern_page_ai import RealSkyvernPageAi
+from skyvern.forge import app
+from skyvern.forge.sdk.core import skyvern_context
+
+
+def ctx() -> skyvern_context.SkyvernContext:
+ skyvern_context.set(skyvern_context.SkyvernContext(organization_id="o_test", workflow_run_id="wr_test"))
+ yield skyvern_context.current()
+ skyvern_context.reset()
+
+
+def test_unknown_writes_full_meta(ctx: skyvern_context.SkyvernContext) -> None:
+ RealSkyvernPageAi._store_classify_result(
+ "UNKNOWN",
+ current_url="https://example.com/search",
+ options={"card_link": "single recipe card"},
+ block_label="search_block",
+ rejection_reasoning="page is search results, not a single card",
+ confidence=1.0,
+ text_excerpt="Recipe results for...",
+ )
+ meta = ctx.last_classify_meta
+ assert meta is not None
+ assert meta["result"] == "UNKNOWN"
+ assert meta["url_at_classify"] == "https://example.com/search"
+ assert meta["block_label_at_classify"] == "search_block"
+ assert meta["candidate_options"] == {"card_link": "single recipe card"}
+ assert meta["rejection_reasoning"] == "page is search results, not a single card"
+ assert meta["confidence"] == 1.0
+ assert meta["text_excerpt"] == "Recipe results for..."
+ assert ctx.last_classify_result == "UNKNOWN"
+
+
+def test_matched_result_clears_meta(ctx: skyvern_context.SkyvernContext) -> None:
+ ctx.last_classify_meta = {"result": "UNKNOWN", "stale": True}
+ RealSkyvernPageAi._store_classify_result(
+ "card_link",
+ current_url="https://example.com/recipe/123",
+ options={"card_link": "single recipe card"},
+ block_label="search_block",
+ )
+ assert ctx.last_classify_meta is None
+ assert ctx.last_classify_result == "card_link"
+
+
+def test_unknown_with_default_args_does_not_raise(ctx: skyvern_context.SkyvernContext) -> None:
+ """LLM-exception path: reasoning='', confidence=0.0, text_excerpt=''."""
+ RealSkyvernPageAi._store_classify_result(
+ "UNKNOWN",
+ current_url="https://example.com",
+ options={"a": "x"},
+ block_label=None,
+ )
+ meta = ctx.last_classify_meta
+ assert meta is not None
+ assert meta["rejection_reasoning"] == ""
+ assert meta["confidence"] == 0.0
+ assert meta["text_excerpt"] == ""
+ assert meta["block_label_at_classify"] is None
+
+
+def test_text_excerpt_is_truncated_to_500(ctx: skyvern_context.SkyvernContext) -> None:
+ long_text = "x" * 1000
+ RealSkyvernPageAi._store_classify_result(
+ "UNKNOWN",
+ current_url="https://example.com",
+ options={"a": "x"},
+ block_label="b",
+ text_excerpt=long_text,
+ )
+ assert ctx.last_classify_meta is not None
+ assert len(ctx.last_classify_meta["text_excerpt"]) == 500
+
+
+def test_no_context_no_raise() -> None:
+ """When there's no current SkyvernContext, the helper is a silent no-op."""
+ RealSkyvernPageAi._store_classify_result(
+ "UNKNOWN",
+ current_url="https://example.com",
+ options={"a": "x"},
+ block_label=None,
+ )
+
+
+def _make_page_stub(url: str = "https://example.com/x") -> SimpleNamespace:
+ async def _inner_text(_selector: str) -> str:
+ return "page text"
+
+ return SimpleNamespace(url=url, inner_text=_inner_text)
+
+
+def _make_page_ai_stub(page: SimpleNamespace) -> RealSkyvernPageAi:
+ inst = RealSkyvernPageAi.__new__(RealSkyvernPageAi)
+ inst.page = page # type: ignore[assignment]
+ inst.scraped_page = SimpleNamespace() # type: ignore[assignment]
+ inst.current_label = "block_a"
+ return inst
+
+
+def fallback_ctx() -> skyvern_context.SkyvernContext:
+ skyvern_context.set(
+ skyvern_context.SkyvernContext(
+ organization_id="o_test",
+ workflow_run_id="wr_test",
+ workflow_permanent_id="wpid_test",
+ )
+ )
+ yield skyvern_context.current()
+ skyvern_context.reset()
+
+
+async def test_ai_element_fallback_consumes_meta_on_success(
+ fallback_ctx: skyvern_context.SkyvernContext, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """Happy path: meta is popped at function entry; subsequent reads see None."""
+ fallback_ctx.last_classify_meta = {
+ "result": "UNKNOWN",
+ "url_at_classify": "https://example.com/x",
+ "block_label_at_classify": "block_a",
+ "candidate_options": {"a": "x"},
+ "rejection_reasoning": "n/a",
+ "confidence": 1.0,
+ "text_excerpt": "",
+ }
+ page = _make_page_stub()
+ page_ai = _make_page_ai_stub(page)
+
+ monkeypatch.setattr(page_ai, "ai_validate", AsyncMock(return_value=True))
+ monkeypatch.setattr(page_ai, "ai_act", AsyncMock(return_value=None))
+ monkeypatch.setattr(app.DATABASE.scripts, "create_fallback_episode", AsyncMock(return_value=None))
+
+ await page_ai.ai_element_fallback(navigation_goal="real goal")
+
+ assert fallback_ctx.last_classify_meta is None
+
+
+async def test_ai_element_fallback_consumes_meta_when_ai_act_raises(
+ fallback_ctx: skyvern_context.SkyvernContext, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """Exception path: meta is popped at function entry, persists None even when ai_act raises mid-loop."""
+ fallback_ctx.last_classify_meta = {
+ "result": "UNKNOWN",
+ "url_at_classify": "https://example.com/x",
+ "block_label_at_classify": "block_a",
+ "candidate_options": {"a": "x"},
+ "rejection_reasoning": "n/a",
+ "confidence": 1.0,
+ "text_excerpt": "",
+ }
+ page = _make_page_stub()
+ page_ai = _make_page_ai_stub(page)
+
+ monkeypatch.setattr(page_ai, "ai_validate", AsyncMock(return_value=False))
+ monkeypatch.setattr(page_ai, "ai_act", AsyncMock(side_effect=RuntimeError("boom")))
+ monkeypatch.setattr(app.DATABASE.scripts, "create_fallback_episode", AsyncMock(return_value=None))
+
+ with pytest.raises(RuntimeError, match="boom"):
+ await page_ai.ai_element_fallback(navigation_goal="real goal", max_steps=2)
+
+ assert fallback_ctx.last_classify_meta is None