Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-12661): route select-verification prompts through LLM_CONFIG_BY_PROMPT_TYPE (#7524)
pedrohsdb6be22e456de6
diff --git a/skyvern/forge/sdk/experimentation/llm_prompt_config.py b/skyvern/forge/sdk/experimentation/llm_prompt_config.pyindex ce4c5f3fa..c7195eb32 100644--- a/skyvern/forge/sdk/experimentation/llm_prompt_config.py+++ b/skyvern/forge/sdk/experimentation/llm_prompt_config.py@@ -165,3 +165,43 @@ async def resolve_check_user_goal_handler(if override is None:return default_handlerreturn LLMAPIHandlerFactory.wrap_for_flex_routing(override)+++async def resolve_prompt_type_handler(+ prompt_type: str,+ distinct_id: str,+ organization_id: str | None,+ default_handler: LLMAPIHandler,+) -> LLMAPIHandler:+ """Return the LLM_CONFIG_BY_PROMPT_TYPE handler for prompt_type if configured; else default_handler."""+ try:+ override = await get_llm_handler_for_prompt_type(prompt_type, distinct_id, organization_id)+ except Exception:+ LOG.warning(+ "Failed to resolve LLM_CONFIG_BY_PROMPT_TYPE; using default handler",+ prompt_type=prompt_type,+ distinct_id=distinct_id,+ organization_id=organization_id,+ exc_info=True,+ )+ return default_handler+ if override is None:+ return default_handler+ return LLMAPIHandlerFactory.wrap_for_flex_routing(override)+++async def resolve_prompt_type_handler_with_override(+ prompt_type: str,+ override_llm_key: str | None,+ distinct_id: str,+ organization_id: str | None,+ default_handler: LLMAPIHandler,+) -> LLMAPIHandler:+ """Payload-first routing unless an explicit override_llm_key opts the call out entirely.++ default_handler must be the call site's full non-payload resolution (e.g. the+ get_override_llm_api_handler(...) result) so both non-payload branches preserve it exactly.+ """+ if override_llm_key:+ return default_handler+ return await resolve_prompt_type_handler(prompt_type, distinct_id, organization_id, default_handler)diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 7c3fe9ef1..969dec298 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -89,7 +89,10 @@ from skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.hashing import diagnostic_fingerprintfrom skyvern.forge.sdk.core.skyvern_context import PendingFileChooserListener, ensure_contextfrom skyvern.forge.sdk.event.factory import EventStrategyFactory-from skyvern.forge.sdk.experimentation.llm_prompt_config import resolve_check_user_goal_handler+from skyvern.forge.sdk.experimentation.llm_prompt_config import (+ resolve_check_user_goal_handler,+ resolve_prompt_type_handler_with_override,+)from skyvern.forge.sdk.experimentation.providers import BaseExperimentationProviderfrom skyvern.forge.sdk.experimentation.slim_llm_output import get_slim_output_template_valuefrom skyvern.forge.sdk.models import Step@@ -6612,7 +6615,13 @@ async def sequentially_select_from_dropdown(select_history=json.dumps(build_sequential_select_history(select_history)),local_datetime=datetime.now(ensure_context().tz_info).isoformat(),)- llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(task.llm_key, default=app.LLM_API_HANDLER)+ llm_api_handler = await resolve_prompt_type_handler_with_override(+ "confirm-multi-selection-finish",+ task.llm_key,+ task.workflow_run_id if task.workflow_run_id else task.task_id,+ task.organization_id,+ LLMAPIHandlerFactory.get_override_llm_api_handler(task.llm_key, default=app.LLM_API_HANDLER),+ )json_response = await llm_api_handler(prompt=prompt, screenshots=[screenshot], step=step, prompt_name="confirm-multi-selection-finish")@@ -7670,8 +7679,7 @@ async def select_from_emerging_elements()LOG.info("Calling LLM to find the match element", sampling=True)- llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(task.llm_key, default=app.LLM_API_HANDLER)- json_response = await llm_api_handler(prompt=prompt, step=step, prompt_name="custom-select")+ json_response = await app.CUSTOM_SELECT_AGENT_LLM_API_HANDLER(prompt=prompt, step=step, prompt_name="custom-select")value: str | None = json_response.get("value", None)LOG.info("LLM response for the matched element",diff --git a/tests/unit/test_llm_prompt_config.py b/tests/unit/test_llm_prompt_config.pyindex bad45f7d2..605458d0f 100644--- a/tests/unit/test_llm_prompt_config.py+++ b/tests/unit/test_llm_prompt_config.py@@ -141,3 +141,83 @@ async def test_prompt_type_in_config_returns_handler(monkeypatch: pytest.MonkeyPresult = await module.get_llm_handler_for_prompt_type("extract-information", "wr_123", "org_1")assert result is fake_handler+++# ---------------------------------------------------------------------------+# resolve_prompt_type_handler: payload-first with default fallback+# ---------------------------------------------------------------------------+++async def test_resolve_prompt_type_handler_returns_flex_wrapped_payload_handler(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ payload_handler = MagicMock(name="payload_handler")+ wrapped_handler = MagicMock(name="wrapped_handler")+ default_handler = MagicMock(name="default_handler")+ monkeypatch.setattr(module, "get_llm_handler_for_prompt_type", AsyncMock(return_value=payload_handler))+ wrap = MagicMock(return_value=wrapped_handler)+ monkeypatch.setattr(module.LLMAPIHandlerFactory, "wrap_for_flex_routing", wrap)++ result = await module.resolve_prompt_type_handler(+ "confirm-multi-selection-finish", "wr_123", "org_1", default_handler+ )++ assert result is wrapped_handler+ wrap.assert_called_once_with(payload_handler)+++async def test_resolve_prompt_type_handler_falls_back_when_unconfigured(monkeypatch: pytest.MonkeyPatch) -> None:+ default_handler = MagicMock(name="default_handler")+ monkeypatch.setattr(module, "get_llm_handler_for_prompt_type", AsyncMock(return_value=None))++ result = await module.resolve_prompt_type_handler("checkbox-verification", "tsk_123", "org_1", default_handler)++ assert result is default_handler+++async def test_resolve_prompt_type_handler_falls_back_on_error(monkeypatch: pytest.MonkeyPatch) -> None:+ default_handler = MagicMock(name="default_handler")+ monkeypatch.setattr(module, "get_llm_handler_for_prompt_type", AsyncMock(side_effect=RuntimeError("posthog down")))+ log, calls = _make_log_recorder()+ monkeypatch.setattr(module, "LOG", log)++ result = await module.resolve_prompt_type_handler("checkbox-verification", "wr_123", None, default_handler)++ assert result is default_handler+ assert [level for level, _ in calls] == ["warning"]+++# ---------------------------------------------------------------------------+# resolve_prompt_type_handler_with_override: explicit key opts out of routing+# ---------------------------------------------------------------------------+++async def test_override_key_skips_prompt_type_resolution(monkeypatch: pytest.MonkeyPatch) -> None:+ default_handler = MagicMock(name="default_handler")+ payload_lookup = AsyncMock(side_effect=AssertionError("payload must not be consulted when override key is set"))+ monkeypatch.setattr(module, "get_llm_handler_for_prompt_type", payload_lookup)++ result = await module.resolve_prompt_type_handler_with_override(+ "confirm-multi-selection-finish", "SOME_EXPLICIT_KEY", "wr_123", "org_1", default_handler+ )++ assert result is default_handler+ payload_lookup.assert_not_awaited()+++async def test_no_override_key_delegates_to_prompt_type_resolution(monkeypatch: pytest.MonkeyPatch) -> None:+ default_handler = MagicMock(name="default_handler")+ resolver = AsyncMock(return_value=default_handler)+ monkeypatch.setattr(module, "resolve_prompt_type_handler", resolver)++ result = await module.resolve_prompt_type_handler_with_override(+ "confirm-multi-selection-finish", None, "wr_123", "org_1", default_handler+ )++ assert result is default_handler+ resolver.assert_awaited_once_with("confirm-multi-selection-finish", "wr_123", "org_1", default_handler)diff --git a/tests/unit/test_select_shadow_match.py b/tests/unit/test_select_shadow_match.pyindex c345dc219..f27a7a7f6 100644--- a/tests/unit/test_select_shadow_match.py+++ b/tests/unit/test_select_shadow_match.py@@ -494,6 +494,7 @@ async def test_emerging_select_call_path_forwards_committed_value_to_shadow_matc"get_override_llm_api_handler",Mock(return_value=llm_handler),)+ monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)monkeypatch.setattr(handler.settings, "SKYVERN_SELECT_SHADOW_MATCH", True)scraped_page = SimpleNamespace(id_to_css_dict={})@@ -597,6 +598,7 @@ async def test_emerging_select_falls_back_to_visible_controlled_listbox_when_no_"get_override_llm_api_handler",Mock(return_value=llm_handler),)+ monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)scraped_page = SimpleNamespace(id_to_css_dict=ids)scraped_page_after_open = SimpleNamespace(id_to_css_dict=ids, element_tree=tree, element_tree_trimmed=tree)@@ -742,6 +744,7 @@ async def test_emerging_select_keeps_off_list_candidate_for_deterministic_match_"get_override_llm_api_handler",Mock(return_value=llm_handler),)+ monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)async def select_deterministically(**kwargs: object) -> tuple[handler.ActionSuccess, str]:get_option_candidates = kwargs["get_option_candidates"]@@ -915,6 +918,7 @@ async def test_emerging_select_rejects_hidden_candidate_from_deterministic_and_l"get_override_llm_api_handler",Mock(return_value=llm_handler),)+ monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)async def select_deterministically(**kwargs: object) -> None:get_option_candidates = kwargs["get_option_candidates"]@@ -976,6 +980,7 @@ async def test_emerging_select_allows_input_text_on_anchor_element("get_override_llm_api_handler",Mock(return_value=llm_handler),)+ monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)scraped_page = SimpleNamespace(id_to_css_dict={"field-id": "[unique_id=field-id]"})scraped_page_after_open = SimpleNamespace(@@ -1029,6 +1034,7 @@ async def test_emerging_select_rejects_input_text_on_element_outside_anchor_and_"get_override_llm_api_handler",Mock(return_value=llm_handler),)+ monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)scraped_page = SimpleNamespace(id_to_css_dict={"field-id": "[unique_id=field-id]"})scraped_page_after_open = SimpleNamespace(@@ -1086,6 +1092,7 @@ async def test_emerging_select_off_list_pick_still_proceeds_when_flag_disabled("get_override_llm_api_handler",Mock(return_value=llm_handler),)+ monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)scraped_page = SimpleNamespace(id_to_css_dict={"field-id": "[unique_id=field-id]"})scraped_page_after_open = SimpleNamespace(