Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-9425): drop hallucinated failure_category codes from user-defined errors (#5775)
Aaron Perez8d7ecce7daa1
diff --git a/skyvern/errors/errors.py b/skyvern/errors/errors.pyindex af07f0693..d1c83e8ab 100644--- a/skyvern/errors/errors.py+++ b/skyvern/errors/errors.py@@ -10,6 +10,30 @@ class UserDefinedError(BaseModel):return f"{self.reasoning}(error_code={self.error_code}, confidence_float={self.confidence_float})"+def filter_to_user_defined_codes(+ errors: list[UserDefinedError],+ error_code_mapping: dict[str, str] | None,+) -> tuple[list[UserDefinedError], list[str]]:+ """Drop LLM-returned errors whose code is not a key in error_code_mapping.++ LLM-based error surfacing prompts occasionally hallucinate codes from the+ failure_categories taxonomy (e.g. LLM_REASONING_ERROR) and place them in+ the user-defined errors field. Returns (kept, dropped_codes) so callers+ can log the drop with their own task/step context.+ """+ if not error_code_mapping:+ return [], [error.error_code for error in errors]+ allowed = set(error_code_mapping.keys())+ kept: list[UserDefinedError] = []+ dropped: list[str] = []+ for error in errors:+ if error.error_code in allowed:+ kept.append(error)+ else:+ dropped.append(error.error_code)+ return kept, dropped++class SkyvernDefinedError(BaseModel):error_code: strreasoning: strdiff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 91e28548b..f880218ef 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -37,6 +37,7 @@ from skyvern.errors.errors import (ReachMaxStepsError,TimeoutGetTOTPVerificationCodeError,UserDefinedError,+ filter_to_user_defined_codes,)from skyvern.exceptions import (BrowserSessionNotFound,@@ -4697,6 +4698,18 @@ class ForgeAgent:)return next_step+ def _filter_response_errors(self, task: Task, step: Step, errors: list[UserDefinedError]) -> list[UserDefinedError]:+ kept, dropped = filter_to_user_defined_codes(errors, task.error_code_mapping)+ if dropped:+ LOG.warning(+ "Dropped LLM-returned error codes not in user error_code_mapping",+ task_id=task.task_id,+ step_id=step.step_id,+ dropped_codes=dropped,+ allowed_codes=sorted((task.error_code_mapping or {}).keys()),+ )+ return kept+async def summary_failure_reason_for_max_steps(self,organization: Organization,@@ -4711,26 +4724,26 @@ class ForgeAgent:steps = await app.DATABASE.tasks.get_task_steps(task_id=task.task_id, organization_id=organization.organization_id)- for step_cnt, step in enumerate(steps):- if step.output is None:+ for step_cnt, cur_step in enumerate(steps):+ if cur_step.output is None:continue- if len(step.output.errors) > 0:- failure_reason = ";".join([repr(err) for err in step.output.errors])+ if len(cur_step.output.errors) > 0:+ failure_reason = ";".join([repr(err) for err in cur_step.output.errors])return MaxStepsReasonResponse(page_info="",reasoning=failure_reason,- errors=step.output.errors,+ errors=cur_step.output.errors,)- if step.output.actions_and_results is None:+ if cur_step.output.actions_and_results is None:continueaction_result_summary: list[str] = []step_result: dict[str, Any] = {"order": step_cnt,}- for action, action_results in step.output.actions_and_results:+ for action, action_results in cur_step.output.actions_and_results:if len(action_results) == 0:continuelast_result = action_results[-1]@@ -4788,7 +4801,9 @@ class ForgeAgent:prompt_name="summarize-max-steps-reason",system_prompt=task.workflow_system_prompt,)- return MaxStepsReasonResponse.model_validate(json_response)+ response = MaxStepsReasonResponse.model_validate(json_response)+ response.errors = self._filter_response_errors(task=task, step=step, errors=response.errors)+ return responseexcept Exception:LOG.warning("Failed to summary the failure reason")# Check if we have LLM errors even if the summarization failed@@ -4931,7 +4946,9 @@ class ForgeAgent:prompt_name="summarize-max-retries-reason",system_prompt=task.workflow_system_prompt,)- return MaxStepsReasonResponse.model_validate(json_response)+ response = MaxStepsReasonResponse.model_validate(json_response)+ response.errors = self._filter_response_errors(task=task, step=step, errors=response.errors)+ return responseexcept Exception:LOG.warning("Failed to summarize the failure reason for max retries")# Check if we have LLM errors even if the summarization faileddiff --git a/skyvern/forge/prompts/skyvern/summarize-max-retries-reason.j2 b/skyvern/forge/prompts/skyvern/summarize-max-retries-reason.j2index f857f0956..95854640f 100644--- a/skyvern/forge/prompts/skyvern/summarize-max-retries-reason.j2+++ b/skyvern/forge/prompts/skyvern/summarize-max-retries-reason.j2@@ -23,7 +23,7 @@ Make sure to ONLY return the JSON object in this format with no additional textUser Goal:{{ navigation_goal }}{% if error_code_mapping_str %}-Use the error codes and their descriptions to surface user-defined errors. Do not return any error that's not defined by the user. User defined errors:+Use the error codes and their descriptions to surface user-defined errors. The `errors` array is ONLY for codes whose keys appear in the JSON below. The `failure_categories` taxonomy (LLM_REASONING_ERROR, ELEMENT_NOT_FOUND, etc.) is separate and must never appear in `errors`. If no user-defined code applies, return an empty `errors` array. User defined errors:{{ error_code_mapping_str }}{% endif %}diff --git a/skyvern/forge/prompts/skyvern/summarize-max-steps-reason.j2 b/skyvern/forge/prompts/skyvern/summarize-max-steps-reason.j2index 33577b6e0..465f3125b 100644--- a/skyvern/forge/prompts/skyvern/summarize-max-steps-reason.j2+++ b/skyvern/forge/prompts/skyvern/summarize-max-steps-reason.j2@@ -22,7 +22,7 @@ Make sure to ONLY return the JSON object in this format with no additional textUser Goal:{{ navigation_goal }}{% if error_code_mapping_str %}-Use the error codes and their descriptions to surface user-defined errors. Do not return any error that's not defined by the user. User defined errors:+Use the error codes and their descriptions to surface user-defined errors. The `errors` array is ONLY for codes whose keys appear in the JSON below. The `failure_categories` taxonomy (LLM_REASONING_ERROR, ELEMENT_NOT_FOUND, etc.) is separate and must never appear in `errors`. If no user-defined code applies, return an empty `errors` array. User defined errors:{{ error_code_mapping_str }}{% endif %}diff --git a/skyvern/services/error_detection_service.py b/skyvern/services/error_detection_service.pyindex e138686dd..9f1e4f13f 100644--- a/skyvern/services/error_detection_service.py+++ b/skyvern/services/error_detection_service.py@@ -13,7 +13,7 @@ from datetime import datetimeimport structlogfrom playwright.async_api import Page-from skyvern.errors.errors import UserDefinedError+from skyvern.errors.errors import UserDefinedError, filter_to_user_defined_codesfrom skyvern.forge import appfrom skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.core import skyvern_context@@ -238,6 +238,16 @@ async def _detect_errors_from_context(exc_info=True,)+ user_defined_errors, dropped = filter_to_user_defined_codes(user_defined_errors, task.error_code_mapping)+ if dropped:+ LOG.warning(+ "Dropped LLM-returned error codes not in user error_code_mapping",+ task_id=task.task_id,+ step_id=step.step_id,+ dropped_codes=dropped,+ allowed_codes=sorted((task.error_code_mapping or {}).keys()),+ )+LOG.info("Detected user-defined errors from context",task_id=task.task_id,diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.pyindex 90e3d08df..c38bb526d 100644--- a/skyvern/services/script_service.py+++ b/skyvern/services/script_service.py@@ -26,7 +26,7 @@ from skyvern.constants import (from skyvern.core.script_generations.constants import SCRIPT_TASK_BLOCKSfrom skyvern.core.script_generations.generate_script import _build_block_fn, create_or_update_script_blockfrom skyvern.core.script_generations.script_skyvern_page import script_run_context_manager-from skyvern.errors.errors import UserDefinedError+from skyvern.errors.errors import UserDefinedError, filter_to_user_defined_codesfrom skyvern.exceptions import (CachedDownloadError,ScriptNotFound,@@ -1041,6 +1041,16 @@ async def _detect_user_defined_errors(error_dict=error_dict,)+ user_defined_errors, dropped = filter_to_user_defined_codes(user_defined_errors, error_code_mapping)+ if dropped:+ LOG.warning(+ "Dropped LLM-returned error codes not in user error_code_mapping",+ task_id=task.task_id,+ step_id=step.step_id,+ dropped_codes=dropped,+ allowed_codes=sorted((error_code_mapping or {}).keys()),+ )+LOG.info("Detected user-defined errors",task_id=task.task_id,diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 9693f31dd..ff43a1125 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -25,7 +25,7 @@ from skyvern.constants import (DROPDOWN_MENU_MAX_DISTANCE,SKYVERN_ID_ATTR,)-from skyvern.errors.errors import TOTPExpiredError+from skyvern.errors.errors import TOTPExpiredError, filter_to_user_defined_codesfrom skyvern.exceptions import (EmptySelect,ErrEmptyTweakValue,@@ -4918,4 +4918,14 @@ async def extract_user_defined_errors(step=step,prompt_name="surface-user-defined-errors",)- return [UserDefinedError.model_validate(error) for error in json_response.get("errors", [])]+ parsed = [UserDefinedError.model_validate(error) for error in json_response.get("errors", [])]+ kept, dropped = filter_to_user_defined_codes(parsed, task.error_code_mapping)+ if dropped:+ LOG.warning(+ "Dropped LLM-returned error codes not in user error_code_mapping",+ task_id=task.task_id,+ step_id=step.step_id,+ dropped_codes=dropped,+ allowed_codes=sorted((task.error_code_mapping or {}).keys()),+ )+ return keptdiff --git a/tests/unit/test_error_detection_integration.py b/tests/unit/test_error_detection_integration.pyindex f53be3ad4..7922780a9 100644--- a/tests/unit/test_error_detection_integration.py+++ b/tests/unit/test_error_detection_integration.py@@ -320,6 +320,40 @@ async def test_error_detection_with_workflow_task(agent, mock_browser_state):assert "workflow_run_id" not in call_kwargs+def test_response_error_filter_drops_hallucinated_failure_category_codes(agent):+ """SKY-9425: LLM summarization can mix failure_categories codes (LLM_REASONING_ERROR)+ into the user-defined errors array. _filter_response_errors must drop any code that+ isn't a key in error_code_mapping so only customer-declared codes leak to the user.+ """+ now = datetime.now()+ organization = make_organization(now)+ task = make_task(+ now,+ organization,+ error_code_mapping={+ "DATA_UNAVAILABLE": "if the required month and year is not present in the dropdown, terminate"+ },+ )+ step = make_step(now, task, step_id="step-9425", status=StepStatus.failed, order=1, output=None)++ hallucinated = [+ UserDefinedError(+ error_code="LLM_REASONING_ERROR",+ reasoning="The LLM misinterpreted the action type",+ confidence_float=0.8,+ ),+ UserDefinedError(+ error_code="DATA_UNAVAILABLE",+ reasoning="April 2026 not in dropdown",+ confidence_float=0.9,+ ),+ ]++ kept = agent._filter_response_errors(task=task, step=step, errors=hallucinated)++ assert [e.error_code for e in kept] == ["DATA_UNAVAILABLE"]++@pytest.mark.asyncioasync def test_error_detection_performance_doesnt_block_failure(agent, mock_browser_state):"""Test that slow error detection doesn't significantly delay task failure."""diff --git a/tests/unit/test_filter_user_defined_codes.py b/tests/unit/test_filter_user_defined_codes.pynew file mode 100644index 000000000..9f9bdf956--- /dev/null+++ b/tests/unit/test_filter_user_defined_codes.py@@ -0,0 +1,69 @@+"""Tests for filter_to_user_defined_codes whitelist filter.++Reproduces SKY-9425: LLM-based summarization prompts can hallucinate codes+from the failure_categories taxonomy (e.g. LLM_REASONING_ERROR) into the+user-defined errors array. The filter drops anything not in the user's+error_code_mapping so only declared codes leak through to the user.+"""++from skyvern.errors.errors import UserDefinedError, filter_to_user_defined_codes+++def _make_error(code: str) -> UserDefinedError:+ return UserDefinedError(error_code=code, reasoning="r", confidence_float=0.8)+++def test_filter_keeps_codes_present_in_mapping() -> None:+ errors = [_make_error("DATA_UNAVAILABLE"), _make_error("OTHER_USER_CODE")]+ mapping = {"DATA_UNAVAILABLE": "x", "OTHER_USER_CODE": "y"}++ kept, dropped = filter_to_user_defined_codes(errors, mapping)++ assert [e.error_code for e in kept] == ["DATA_UNAVAILABLE", "OTHER_USER_CODE"]+ assert dropped == []+++def test_filter_drops_failure_category_codes_not_in_mapping() -> None:+ errors = [_make_error("LLM_REASONING_ERROR"), _make_error("DATA_UNAVAILABLE")]+ mapping = {"DATA_UNAVAILABLE": "if month/year not in dropdown, terminate"}++ kept, dropped = filter_to_user_defined_codes(errors, mapping)++ assert [e.error_code for e in kept] == ["DATA_UNAVAILABLE"]+ assert dropped == ["LLM_REASONING_ERROR"]+++def test_filter_returns_empty_when_mapping_is_none() -> None:+ errors = [_make_error("ANYTHING")]++ kept, dropped = filter_to_user_defined_codes(errors, None)++ assert kept == []+ assert dropped == ["ANYTHING"]+++def test_filter_returns_empty_when_mapping_is_empty_dict() -> None:+ errors = [_make_error("ANYTHING")]++ kept, dropped = filter_to_user_defined_codes(errors, {})++ assert kept == []+ assert dropped == ["ANYTHING"]+++def test_filter_handles_empty_errors_input() -> None:+ kept, dropped = filter_to_user_defined_codes([], {"FOO": "bar"})++ assert kept == []+ assert dropped == []+++def test_filter_is_case_sensitive() -> None:+ """Codes must match exactly — mapping is case-sensitive by design."""+ errors = [_make_error("data_unavailable")]+ mapping = {"DATA_UNAVAILABLE": "x"}++ kept, dropped = filter_to_user_defined_codes(errors, mapping)++ assert kept == []+ assert dropped == ["data_unavailable"]