Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-9622): suppress false-positive warning when prompt type absent from PostHog config (#5849)
Aaron Perez055544fa897b
diff --git a/skyvern/forge/sdk/experimentation/llm_prompt_config.py b/skyvern/forge/sdk/experimentation/llm_prompt_config.pyindex bcf380bf6..7da391592 100644--- a/skyvern/forge/sdk/experimentation/llm_prompt_config.py+++ b/skyvern/forge/sdk/experimentation/llm_prompt_config.py@@ -59,11 +59,12 @@ async def get_llm_handler_for_prompt_type() -> LLMAPIHandler | None:"""Return initialized handler for prompt type from LLM_CONFIG_BY_PROMPT_TYPE flag."""config = await get_llm_config_by_prompt_type(distinct_id, organization_id)- if not config or prompt_type not in config:- LOG.warning(- "No config found for prompt type",+ if not config:+ return None+ if prompt_type not in config:+ LOG.debug(+ "Prompt type not in LLM config, using default handler",prompt_type=prompt_type,- config=config,distinct_id=distinct_id,organization_id=organization_id,)diff --git a/tests/unit/test_llm_prompt_config.py b/tests/unit/test_llm_prompt_config.pynew file mode 100644index 000000000..d27f4c9ad--- /dev/null+++ b/tests/unit/test_llm_prompt_config.py@@ -0,0 +1,121 @@+"""Tests for SKY-9622: get_llm_handler_for_prompt_type warning behaviour.++Verified contracts:+- When config is None (flag disabled or error): return None silently.+- When config exists but prompt type is absent: return None at debug level only.+- When config has the prompt type: delegate to LLMAPIHandlerFactory.+"""++from __future__ import annotations++from typing import Any+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from skyvern.forge.sdk.experimentation import llm_prompt_config as module++# ---------------------------------------------------------------------------+# helpers+# ---------------------------------------------------------------------------+++def _make_log_recorder() -> tuple[MagicMock, list[tuple[str, dict[str, Any]]]]:+ """Return a mock LOG and a list that accumulates (level, kwargs) tuples."""+ calls: list[tuple[str, dict[str, Any]]] = []+ log = MagicMock()++ def _record(level: str) -> Any:+ def _call(*args: object, **kwargs: object) -> None:+ calls.append((level, dict(kwargs)))++ return _call++ log.warning = _record("warning")+ log.debug = _record("debug")+ log.info = _record("info")+ log.error = _record("error")+ return log, calls+++# ---------------------------------------------------------------------------+# get_llm_handler_for_prompt_type: no-config path+# ---------------------------------------------------------------------------+++async def test_no_config_returns_none_without_warning(monkeypatch: pytest.MonkeyPatch) -> None:+ """Flag disabled → config is None → return None, no LOG.warning."""+ monkeypatch.setattr(+ module,+ "get_llm_config_by_prompt_type",+ AsyncMock(return_value=None),+ )+ log, calls = _make_log_recorder()+ monkeypatch.setattr(module, "LOG", log)++ result = await module.get_llm_handler_for_prompt_type("extract-information", "wr_123", "org_1")++ assert result is None+ warning_calls = [c for level, c in calls if level == "warning"]+ assert not warning_calls, f"Expected no warnings, got: {warning_calls}"+++# ---------------------------------------------------------------------------+# get_llm_handler_for_prompt_type: prompt type absent from config+# ---------------------------------------------------------------------------+++async def test_prompt_type_missing_from_config_returns_none_without_warning(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Config exists but prompt type not present → return None, no LOG.warning.++ The four types absent from the PostHog payload at time of SKY-9622:+ single-input-action, single-click-action, single-select-action, custom-select.+ These fall back to the default app handler; warning was incorrect.+ """+ config = {+ "extract-information": "GEMINI_2_5_FLASH_WITH_FALLBACK",+ "normal-select": "GEMINI_2_5_FLASH_WITH_FALLBACK",+ }+ monkeypatch.setattr(+ module,+ "get_llm_config_by_prompt_type",+ AsyncMock(return_value=config),+ )+ log, calls = _make_log_recorder()+ monkeypatch.setattr(module, "LOG", log)++ for prompt_type in ("single-input-action", "single-click-action", "single-select-action", "custom-select"):+ calls.clear()+ result = await module.get_llm_handler_for_prompt_type(prompt_type, "wr_123", "org_1")+ assert result is None, f"{prompt_type}: expected None"+ warning_calls = [c for level, c in calls if level == "warning"]+ assert not warning_calls, f"{prompt_type}: unexpected warnings: {warning_calls}"+++# ---------------------------------------------------------------------------+# get_llm_handler_for_prompt_type: happy path (type present in config)+# ---------------------------------------------------------------------------+++async def test_prompt_type_in_config_returns_handler(monkeypatch: pytest.MonkeyPatch) -> None:+ """Config contains prompt type → LLMAPIHandlerFactory is called, handler returned."""+ config = {"extract-information": "GEMINI_2_5_FLASH_WITH_FALLBACK"}+ monkeypatch.setattr(+ module,+ "get_llm_config_by_prompt_type",+ AsyncMock(return_value=config),+ )++ fake_handler = object()+ with patch(+ "skyvern.forge.sdk.experimentation.llm_prompt_config.LLMAPIHandlerFactory.get_llm_api_handler",+ return_value=fake_handler,+ ):+ result = await module.get_llm_handler_for_prompt_type("extract-information", "wr_123", "org_1")++ assert result is fake_handler