Skip to content

Automate browser based workflows with AI

fix(SKY-12661): route select-verification prompts through LLM_CONFIG_BY_PROMPT_TYPE (#7524)

pedrohsdb6be22e456de6
diff --git a/skyvern/forge/sdk/experimentation/llm_prompt_config.py b/skyvern/forge/sdk/experimentation/llm_prompt_config.py
index ce4c5f3fa..c7195eb32 100644
--- a/skyvern/forge/sdk/experimentation/llm_prompt_config.py
+++ b/skyvern/forge/sdk/experimentation/llm_prompt_config.py
@@ -165,3 +165,43 @@ async def resolve_check_user_goal_handler(
if override is None:
return default_handler
return LLMAPIHandlerFactory.wrap_for_flex_routing(override)
+
+
+async def resolve_prompt_type_handler(
+ prompt_type: str,
+ distinct_id: str,
+ organization_id: str | None,
+ default_handler: LLMAPIHandler,
+) -> LLMAPIHandler:
+ """Return the LLM_CONFIG_BY_PROMPT_TYPE handler for prompt_type if configured; else default_handler."""
+ try:
+ override = await get_llm_handler_for_prompt_type(prompt_type, distinct_id, organization_id)
+ except Exception:
+ LOG.warning(
+ "Failed to resolve LLM_CONFIG_BY_PROMPT_TYPE; using default handler",
+ prompt_type=prompt_type,
+ distinct_id=distinct_id,
+ organization_id=organization_id,
+ exc_info=True,
+ )
+ return default_handler
+ if override is None:
+ return default_handler
+ return LLMAPIHandlerFactory.wrap_for_flex_routing(override)
+
+
+async def resolve_prompt_type_handler_with_override(
+ prompt_type: str,
+ override_llm_key: str | None,
+ distinct_id: str,
+ organization_id: str | None,
+ default_handler: LLMAPIHandler,
+) -> LLMAPIHandler:
+ """Payload-first routing unless an explicit override_llm_key opts the call out entirely.
+
+ default_handler must be the call site's full non-payload resolution (e.g. the
+ get_override_llm_api_handler(...) result) so both non-payload branches preserve it exactly.
+ """
+ if override_llm_key:
+ return default_handler
+ return await resolve_prompt_type_handler(prompt_type, distinct_id, organization_id, default_handler)
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 7c3fe9ef1..969dec298 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -89,7 +89,10 @@ from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.hashing import diagnostic_fingerprint
from skyvern.forge.sdk.core.skyvern_context import PendingFileChooserListener, ensure_context
from skyvern.forge.sdk.event.factory import EventStrategyFactory
-from skyvern.forge.sdk.experimentation.llm_prompt_config import resolve_check_user_goal_handler
+from skyvern.forge.sdk.experimentation.llm_prompt_config import (
+ resolve_check_user_goal_handler,
+ resolve_prompt_type_handler_with_override,
+)
from skyvern.forge.sdk.experimentation.providers import BaseExperimentationProvider
from skyvern.forge.sdk.experimentation.slim_llm_output import get_slim_output_template_value
from skyvern.forge.sdk.models import Step
@@ -6612,7 +6615,13 @@ async def sequentially_select_from_dropdown(
select_history=json.dumps(build_sequential_select_history(select_history)),
local_datetime=datetime.now(ensure_context().tz_info).isoformat(),
)
- llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(task.llm_key, default=app.LLM_API_HANDLER)
+ llm_api_handler = await resolve_prompt_type_handler_with_override(
+ "confirm-multi-selection-finish",
+ task.llm_key,
+ task.workflow_run_id if task.workflow_run_id else task.task_id,
+ task.organization_id,
+ LLMAPIHandlerFactory.get_override_llm_api_handler(task.llm_key, default=app.LLM_API_HANDLER),
+ )
json_response = await llm_api_handler(
prompt=prompt, screenshots=[screenshot], step=step, prompt_name="confirm-multi-selection-finish"
)
@@ -7670,8 +7679,7 @@ async def select_from_emerging_elements(
)
LOG.info("Calling LLM to find the match element", sampling=True)
- llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(task.llm_key, default=app.LLM_API_HANDLER)
- json_response = await llm_api_handler(prompt=prompt, step=step, prompt_name="custom-select")
+ json_response = await app.CUSTOM_SELECT_AGENT_LLM_API_HANDLER(prompt=prompt, step=step, prompt_name="custom-select")
value: str | None = json_response.get("value", None)
LOG.info(
"LLM response for the matched element",
diff --git a/tests/unit/test_llm_prompt_config.py b/tests/unit/test_llm_prompt_config.py
index bad45f7d2..605458d0f 100644
--- a/tests/unit/test_llm_prompt_config.py
+++ b/tests/unit/test_llm_prompt_config.py
@@ -141,3 +141,83 @@ async def test_prompt_type_in_config_returns_handler(monkeypatch: pytest.MonkeyP
result = await module.get_llm_handler_for_prompt_type("extract-information", "wr_123", "org_1")
assert result is fake_handler
+
+
+# ---------------------------------------------------------------------------
+# resolve_prompt_type_handler: payload-first with default fallback
+# ---------------------------------------------------------------------------
+
+
+async def test_resolve_prompt_type_handler_returns_flex_wrapped_payload_handler(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ payload_handler = MagicMock(name="payload_handler")
+ wrapped_handler = MagicMock(name="wrapped_handler")
+ default_handler = MagicMock(name="default_handler")
+ monkeypatch.setattr(module, "get_llm_handler_for_prompt_type", AsyncMock(return_value=payload_handler))
+ wrap = MagicMock(return_value=wrapped_handler)
+ monkeypatch.setattr(module.LLMAPIHandlerFactory, "wrap_for_flex_routing", wrap)
+
+ result = await module.resolve_prompt_type_handler(
+ "confirm-multi-selection-finish", "wr_123", "org_1", default_handler
+ )
+
+ assert result is wrapped_handler
+ wrap.assert_called_once_with(payload_handler)
+
+
+async def test_resolve_prompt_type_handler_falls_back_when_unconfigured(monkeypatch: pytest.MonkeyPatch) -> None:
+ default_handler = MagicMock(name="default_handler")
+ monkeypatch.setattr(module, "get_llm_handler_for_prompt_type", AsyncMock(return_value=None))
+
+ result = await module.resolve_prompt_type_handler("checkbox-verification", "tsk_123", "org_1", default_handler)
+
+ assert result is default_handler
+
+
+async def test_resolve_prompt_type_handler_falls_back_on_error(monkeypatch: pytest.MonkeyPatch) -> None:
+ default_handler = MagicMock(name="default_handler")
+ monkeypatch.setattr(module, "get_llm_handler_for_prompt_type", AsyncMock(side_effect=RuntimeError("posthog down")))
+ log, calls = _make_log_recorder()
+ monkeypatch.setattr(module, "LOG", log)
+
+ result = await module.resolve_prompt_type_handler("checkbox-verification", "wr_123", None, default_handler)
+
+ assert result is default_handler
+ assert [level for level, _ in calls] == ["warning"]
+
+
+# ---------------------------------------------------------------------------
+# resolve_prompt_type_handler_with_override: explicit key opts out of routing
+# ---------------------------------------------------------------------------
+
+
+async def test_override_key_skips_prompt_type_resolution(monkeypatch: pytest.MonkeyPatch) -> None:
+ default_handler = MagicMock(name="default_handler")
+ payload_lookup = AsyncMock(side_effect=AssertionError("payload must not be consulted when override key is set"))
+ monkeypatch.setattr(module, "get_llm_handler_for_prompt_type", payload_lookup)
+
+ result = await module.resolve_prompt_type_handler_with_override(
+ "confirm-multi-selection-finish", "SOME_EXPLICIT_KEY", "wr_123", "org_1", default_handler
+ )
+
+ assert result is default_handler
+ payload_lookup.assert_not_awaited()
+
+
+async def test_no_override_key_delegates_to_prompt_type_resolution(monkeypatch: pytest.MonkeyPatch) -> None:
+ default_handler = MagicMock(name="default_handler")
+ resolver = AsyncMock(return_value=default_handler)
+ monkeypatch.setattr(module, "resolve_prompt_type_handler", resolver)
+
+ result = await module.resolve_prompt_type_handler_with_override(
+ "confirm-multi-selection-finish", None, "wr_123", "org_1", default_handler
+ )
+
+ assert result is default_handler
+ resolver.assert_awaited_once_with("confirm-multi-selection-finish", "wr_123", "org_1", default_handler)
diff --git a/tests/unit/test_select_shadow_match.py b/tests/unit/test_select_shadow_match.py
index c345dc219..f27a7a7f6 100644
--- a/tests/unit/test_select_shadow_match.py
+++ b/tests/unit/test_select_shadow_match.py
@@ -494,6 +494,7 @@ async def test_emerging_select_call_path_forwards_committed_value_to_shadow_matc
"get_override_llm_api_handler",
Mock(return_value=llm_handler),
)
+ monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)
monkeypatch.setattr(handler.settings, "SKYVERN_SELECT_SHADOW_MATCH", True)
scraped_page = SimpleNamespace(id_to_css_dict={})
@@ -597,6 +598,7 @@ async def test_emerging_select_falls_back_to_visible_controlled_listbox_when_no_
"get_override_llm_api_handler",
Mock(return_value=llm_handler),
)
+ monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)
scraped_page = SimpleNamespace(id_to_css_dict=ids)
scraped_page_after_open = SimpleNamespace(id_to_css_dict=ids, element_tree=tree, element_tree_trimmed=tree)
@@ -742,6 +744,7 @@ async def test_emerging_select_keeps_off_list_candidate_for_deterministic_match_
"get_override_llm_api_handler",
Mock(return_value=llm_handler),
)
+ monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)
async def select_deterministically(**kwargs: object) -> tuple[handler.ActionSuccess, str]:
get_option_candidates = kwargs["get_option_candidates"]
@@ -915,6 +918,7 @@ async def test_emerging_select_rejects_hidden_candidate_from_deterministic_and_l
"get_override_llm_api_handler",
Mock(return_value=llm_handler),
)
+ monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)
async def select_deterministically(**kwargs: object) -> None:
get_option_candidates = kwargs["get_option_candidates"]
@@ -976,6 +980,7 @@ async def test_emerging_select_allows_input_text_on_anchor_element(
"get_override_llm_api_handler",
Mock(return_value=llm_handler),
)
+ monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)
scraped_page = SimpleNamespace(id_to_css_dict={"field-id": "[unique_id=field-id]"})
scraped_page_after_open = SimpleNamespace(
@@ -1029,6 +1034,7 @@ async def test_emerging_select_rejects_input_text_on_element_outside_anchor_and_
"get_override_llm_api_handler",
Mock(return_value=llm_handler),
)
+ monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)
scraped_page = SimpleNamespace(id_to_css_dict={"field-id": "[unique_id=field-id]"})
scraped_page_after_open = SimpleNamespace(
@@ -1086,6 +1092,7 @@ async def test_emerging_select_off_list_pick_still_proceeds_when_flag_disabled(
"get_override_llm_api_handler",
Mock(return_value=llm_handler),
)
+ monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)
scraped_page = SimpleNamespace(id_to_css_dict={"field-id": "[unique_id=field-id]"})
scraped_page_after_open = SimpleNamespace(