Skip to content

Automate browser based workflows with AI

fix(SKY-9425): drop hallucinated failure_category codes from user-defined errors (#5775)

Aaron Perez8d7ecce7daa1
diff --git a/skyvern/errors/errors.py b/skyvern/errors/errors.py
index af07f0693..d1c83e8ab 100644
--- a/skyvern/errors/errors.py
+++ b/skyvern/errors/errors.py
@@ -10,6 +10,30 @@ class UserDefinedError(BaseModel):
return f"{self.reasoning}(error_code={self.error_code}, confidence_float={self.confidence_float})"
+def filter_to_user_defined_codes(
+ errors: list[UserDefinedError],
+ error_code_mapping: dict[str, str] | None,
+) -> tuple[list[UserDefinedError], list[str]]:
+ """Drop LLM-returned errors whose code is not a key in error_code_mapping.
+
+ LLM-based error surfacing prompts occasionally hallucinate codes from the
+ failure_categories taxonomy (e.g. LLM_REASONING_ERROR) and place them in
+ the user-defined errors field. Returns (kept, dropped_codes) so callers
+ can log the drop with their own task/step context.
+ """
+ if not error_code_mapping:
+ return [], [error.error_code for error in errors]
+ allowed = set(error_code_mapping.keys())
+ kept: list[UserDefinedError] = []
+ dropped: list[str] = []
+ for error in errors:
+ if error.error_code in allowed:
+ kept.append(error)
+ else:
+ dropped.append(error.error_code)
+ return kept, dropped
+
+
class SkyvernDefinedError(BaseModel):
error_code: str
reasoning: str
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 91e28548b..f880218ef 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -37,6 +37,7 @@ from skyvern.errors.errors import (
ReachMaxStepsError,
TimeoutGetTOTPVerificationCodeError,
UserDefinedError,
+ filter_to_user_defined_codes,
)
from skyvern.exceptions import (
BrowserSessionNotFound,
@@ -4697,6 +4698,18 @@ class ForgeAgent:
)
return next_step
+ def _filter_response_errors(self, task: Task, step: Step, errors: list[UserDefinedError]) -> list[UserDefinedError]:
+ kept, dropped = filter_to_user_defined_codes(errors, task.error_code_mapping)
+ if dropped:
+ LOG.warning(
+ "Dropped LLM-returned error codes not in user error_code_mapping",
+ task_id=task.task_id,
+ step_id=step.step_id,
+ dropped_codes=dropped,
+ allowed_codes=sorted((task.error_code_mapping or {}).keys()),
+ )
+ return kept
+
async def summary_failure_reason_for_max_steps(
self,
organization: Organization,
@@ -4711,26 +4724,26 @@ class ForgeAgent:
steps = await app.DATABASE.tasks.get_task_steps(
task_id=task.task_id, organization_id=organization.organization_id
)
- for step_cnt, step in enumerate(steps):
- if step.output is None:
+ for step_cnt, cur_step in enumerate(steps):
+ if cur_step.output is None:
continue
- if len(step.output.errors) > 0:
- failure_reason = ";".join([repr(err) for err in step.output.errors])
+ if len(cur_step.output.errors) > 0:
+ failure_reason = ";".join([repr(err) for err in cur_step.output.errors])
return MaxStepsReasonResponse(
page_info="",
reasoning=failure_reason,
- errors=step.output.errors,
+ errors=cur_step.output.errors,
)
- if step.output.actions_and_results is None:
+ if cur_step.output.actions_and_results is None:
continue
action_result_summary: list[str] = []
step_result: dict[str, Any] = {
"order": step_cnt,
}
- for action, action_results in step.output.actions_and_results:
+ for action, action_results in cur_step.output.actions_and_results:
if len(action_results) == 0:
continue
last_result = action_results[-1]
@@ -4788,7 +4801,9 @@ class ForgeAgent:
prompt_name="summarize-max-steps-reason",
system_prompt=task.workflow_system_prompt,
)
- return MaxStepsReasonResponse.model_validate(json_response)
+ response = MaxStepsReasonResponse.model_validate(json_response)
+ response.errors = self._filter_response_errors(task=task, step=step, errors=response.errors)
+ return response
except Exception:
LOG.warning("Failed to summary the failure reason")
# Check if we have LLM errors even if the summarization failed
@@ -4931,7 +4946,9 @@ class ForgeAgent:
prompt_name="summarize-max-retries-reason",
system_prompt=task.workflow_system_prompt,
)
- return MaxStepsReasonResponse.model_validate(json_response)
+ response = MaxStepsReasonResponse.model_validate(json_response)
+ response.errors = self._filter_response_errors(task=task, step=step, errors=response.errors)
+ return response
except Exception:
LOG.warning("Failed to summarize the failure reason for max retries")
# Check if we have LLM errors even if the summarization failed
diff --git a/skyvern/forge/prompts/skyvern/summarize-max-retries-reason.j2 b/skyvern/forge/prompts/skyvern/summarize-max-retries-reason.j2
index f857f0956..95854640f 100644
--- a/skyvern/forge/prompts/skyvern/summarize-max-retries-reason.j2
+++ b/skyvern/forge/prompts/skyvern/summarize-max-retries-reason.j2
@@ -23,7 +23,7 @@ Make sure to ONLY return the JSON object in this format with no additional text
User Goal:
{{ navigation_goal }}
{% if error_code_mapping_str %}
-Use the error codes and their descriptions to surface user-defined errors. Do not return any error that's not defined by the user. User defined errors:
+Use the error codes and their descriptions to surface user-defined errors. The `errors` array is ONLY for codes whose keys appear in the JSON below. The `failure_categories` taxonomy (LLM_REASONING_ERROR, ELEMENT_NOT_FOUND, etc.) is separate and must never appear in `errors`. If no user-defined code applies, return an empty `errors` array. User defined errors:
{{ error_code_mapping_str }}
{% endif %}
diff --git a/skyvern/forge/prompts/skyvern/summarize-max-steps-reason.j2 b/skyvern/forge/prompts/skyvern/summarize-max-steps-reason.j2
index 33577b6e0..465f3125b 100644
--- a/skyvern/forge/prompts/skyvern/summarize-max-steps-reason.j2
+++ b/skyvern/forge/prompts/skyvern/summarize-max-steps-reason.j2
@@ -22,7 +22,7 @@ Make sure to ONLY return the JSON object in this format with no additional text
User Goal:
{{ navigation_goal }}
{% if error_code_mapping_str %}
-Use the error codes and their descriptions to surface user-defined errors. Do not return any error that's not defined by the user. User defined errors:
+Use the error codes and their descriptions to surface user-defined errors. The `errors` array is ONLY for codes whose keys appear in the JSON below. The `failure_categories` taxonomy (LLM_REASONING_ERROR, ELEMENT_NOT_FOUND, etc.) is separate and must never appear in `errors`. If no user-defined code applies, return an empty `errors` array. User defined errors:
{{ error_code_mapping_str }}
{% endif %}
diff --git a/skyvern/services/error_detection_service.py b/skyvern/services/error_detection_service.py
index e138686dd..9f1e4f13f 100644
--- a/skyvern/services/error_detection_service.py
+++ b/skyvern/services/error_detection_service.py
@@ -13,7 +13,7 @@ from datetime import datetime
import structlog
from playwright.async_api import Page
-from skyvern.errors.errors import UserDefinedError
+from skyvern.errors.errors import UserDefinedError, filter_to_user_defined_codes
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.core import skyvern_context
@@ -238,6 +238,16 @@ async def _detect_errors_from_context(
exc_info=True,
)
+ user_defined_errors, dropped = filter_to_user_defined_codes(user_defined_errors, task.error_code_mapping)
+ if dropped:
+ LOG.warning(
+ "Dropped LLM-returned error codes not in user error_code_mapping",
+ task_id=task.task_id,
+ step_id=step.step_id,
+ dropped_codes=dropped,
+ allowed_codes=sorted((task.error_code_mapping or {}).keys()),
+ )
+
LOG.info(
"Detected user-defined errors from context",
task_id=task.task_id,
diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.py
index 90e3d08df..c38bb526d 100644
--- a/skyvern/services/script_service.py
+++ b/skyvern/services/script_service.py
@@ -26,7 +26,7 @@ from skyvern.constants import (
from skyvern.core.script_generations.constants import SCRIPT_TASK_BLOCKS
from skyvern.core.script_generations.generate_script import _build_block_fn, create_or_update_script_block
from skyvern.core.script_generations.script_skyvern_page import script_run_context_manager
-from skyvern.errors.errors import UserDefinedError
+from skyvern.errors.errors import UserDefinedError, filter_to_user_defined_codes
from skyvern.exceptions import (
CachedDownloadError,
ScriptNotFound,
@@ -1041,6 +1041,16 @@ async def _detect_user_defined_errors(
error_dict=error_dict,
)
+ user_defined_errors, dropped = filter_to_user_defined_codes(user_defined_errors, error_code_mapping)
+ if dropped:
+ LOG.warning(
+ "Dropped LLM-returned error codes not in user error_code_mapping",
+ task_id=task.task_id,
+ step_id=step.step_id,
+ dropped_codes=dropped,
+ allowed_codes=sorted((error_code_mapping or {}).keys()),
+ )
+
LOG.info(
"Detected user-defined errors",
task_id=task.task_id,
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 9693f31dd..ff43a1125 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -25,7 +25,7 @@ from skyvern.constants import (
DROPDOWN_MENU_MAX_DISTANCE,
SKYVERN_ID_ATTR,
)
-from skyvern.errors.errors import TOTPExpiredError
+from skyvern.errors.errors import TOTPExpiredError, filter_to_user_defined_codes
from skyvern.exceptions import (
EmptySelect,
ErrEmptyTweakValue,
@@ -4918,4 +4918,14 @@ async def extract_user_defined_errors(
step=step,
prompt_name="surface-user-defined-errors",
)
- return [UserDefinedError.model_validate(error) for error in json_response.get("errors", [])]
+ parsed = [UserDefinedError.model_validate(error) for error in json_response.get("errors", [])]
+ kept, dropped = filter_to_user_defined_codes(parsed, task.error_code_mapping)
+ if dropped:
+ LOG.warning(
+ "Dropped LLM-returned error codes not in user error_code_mapping",
+ task_id=task.task_id,
+ step_id=step.step_id,
+ dropped_codes=dropped,
+ allowed_codes=sorted((task.error_code_mapping or {}).keys()),
+ )
+ return kept
diff --git a/tests/unit/test_error_detection_integration.py b/tests/unit/test_error_detection_integration.py
index f53be3ad4..7922780a9 100644
--- a/tests/unit/test_error_detection_integration.py
+++ b/tests/unit/test_error_detection_integration.py
@@ -320,6 +320,40 @@ async def test_error_detection_with_workflow_task(agent, mock_browser_state):
assert "workflow_run_id" not in call_kwargs
+def test_response_error_filter_drops_hallucinated_failure_category_codes(agent):
+ """SKY-9425: LLM summarization can mix failure_categories codes (LLM_REASONING_ERROR)
+ into the user-defined errors array. _filter_response_errors must drop any code that
+ isn't a key in error_code_mapping so only customer-declared codes leak to the user.
+ """
+ now = datetime.now()
+ organization = make_organization(now)
+ task = make_task(
+ now,
+ organization,
+ error_code_mapping={
+ "DATA_UNAVAILABLE": "if the required month and year is not present in the dropdown, terminate"
+ },
+ )
+ step = make_step(now, task, step_id="step-9425", status=StepStatus.failed, order=1, output=None)
+
+ hallucinated = [
+ UserDefinedError(
+ error_code="LLM_REASONING_ERROR",
+ reasoning="The LLM misinterpreted the action type",
+ confidence_float=0.8,
+ ),
+ UserDefinedError(
+ error_code="DATA_UNAVAILABLE",
+ reasoning="April 2026 not in dropdown",
+ confidence_float=0.9,
+ ),
+ ]
+
+ kept = agent._filter_response_errors(task=task, step=step, errors=hallucinated)
+
+ assert [e.error_code for e in kept] == ["DATA_UNAVAILABLE"]
+
+
@pytest.mark.asyncio
async def test_error_detection_performance_doesnt_block_failure(agent, mock_browser_state):
"""Test that slow error detection doesn't significantly delay task failure."""
diff --git a/tests/unit/test_filter_user_defined_codes.py b/tests/unit/test_filter_user_defined_codes.py
new file mode 100644
index 000000000..9f9bdf956
--- /dev/null
+++ b/tests/unit/test_filter_user_defined_codes.py
@@ -0,0 +1,69 @@
+"""Tests for filter_to_user_defined_codes whitelist filter.
+
+Reproduces SKY-9425: LLM-based summarization prompts can hallucinate codes
+from the failure_categories taxonomy (e.g. LLM_REASONING_ERROR) into the
+user-defined errors array. The filter drops anything not in the user's
+error_code_mapping so only declared codes leak through to the user.
+"""
+
+from skyvern.errors.errors import UserDefinedError, filter_to_user_defined_codes
+
+
+def _make_error(code: str) -> UserDefinedError:
+ return UserDefinedError(error_code=code, reasoning="r", confidence_float=0.8)
+
+
+def test_filter_keeps_codes_present_in_mapping() -> None:
+ errors = [_make_error("DATA_UNAVAILABLE"), _make_error("OTHER_USER_CODE")]
+ mapping = {"DATA_UNAVAILABLE": "x", "OTHER_USER_CODE": "y"}
+
+ kept, dropped = filter_to_user_defined_codes(errors, mapping)
+
+ assert [e.error_code for e in kept] == ["DATA_UNAVAILABLE", "OTHER_USER_CODE"]
+ assert dropped == []
+
+
+def test_filter_drops_failure_category_codes_not_in_mapping() -> None:
+ errors = [_make_error("LLM_REASONING_ERROR"), _make_error("DATA_UNAVAILABLE")]
+ mapping = {"DATA_UNAVAILABLE": "if month/year not in dropdown, terminate"}
+
+ kept, dropped = filter_to_user_defined_codes(errors, mapping)
+
+ assert [e.error_code for e in kept] == ["DATA_UNAVAILABLE"]
+ assert dropped == ["LLM_REASONING_ERROR"]
+
+
+def test_filter_returns_empty_when_mapping_is_none() -> None:
+ errors = [_make_error("ANYTHING")]
+
+ kept, dropped = filter_to_user_defined_codes(errors, None)
+
+ assert kept == []
+ assert dropped == ["ANYTHING"]
+
+
+def test_filter_returns_empty_when_mapping_is_empty_dict() -> None:
+ errors = [_make_error("ANYTHING")]
+
+ kept, dropped = filter_to_user_defined_codes(errors, {})
+
+ assert kept == []
+ assert dropped == ["ANYTHING"]
+
+
+def test_filter_handles_empty_errors_input() -> None:
+ kept, dropped = filter_to_user_defined_codes([], {"FOO": "bar"})
+
+ assert kept == []
+ assert dropped == []
+
+
+def test_filter_is_case_sensitive() -> None:
+ """Codes must match exactly — mapping is case-sensitive by design."""
+ errors = [_make_error("data_unavailable")]
+ mapping = {"DATA_UNAVAILABLE": "x"}
+
+ kept, dropped = filter_to_user_defined_codes(errors, mapping)
+
+ assert kept == []
+ assert dropped == ["data_unavailable"]