Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-11571): repair main CI — add resolve_feature_enabled_unrecorded to 4 missed provider fakes (#7295)
Aaron Perez42484b20fef7
diff --git a/skyvern/forge/sdk/experimentation/providers.py b/skyvern/forge/sdk/experimentation/providers.pyindex c2e0ee25b..2b2e273f7 100644--- a/skyvern/forge/sdk/experimentation/providers.py+++ b/skyvern/forge/sdk/experimentation/providers.py@@ -188,6 +188,13 @@ class BaseExperimentationProvider(ABC):)return feature_flag_value+ async def resolve_feature_enabled_unrecorded(+ self, feature_name: str, distinct_id: str, properties: dict | None = None+ ) -> bool:+ """Resolve a flag while leaving attribution and logging to the caller."""+ await self._prepare_feature_flag_resolution(feature_name, cached=True)+ return await self._is_feature_enabled(feature_name, distinct_id, properties)+async def is_feature_enabled_cached(self, feature_name: str, distinct_id: str, properties: dict | None = None) -> bool:diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 1db35ef02..1fa1a947b 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -13,6 +13,7 @@ from pathlib import Pathfrom typing import Any, Awaitable, Callable, List, TypedDictimport structlog+from cachetools import TTLCachefrom fuzzysearch import find_near_matchesfrom opentelemetry import trace as otel_tracefrom playwright._impl._errors import Error as PlaywrightError@@ -84,6 +85,7 @@ from skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import PendingFileChooserListener, ensure_contextfrom skyvern.forge.sdk.event.factory import EventStrategyFactoryfrom skyvern.forge.sdk.experimentation.llm_prompt_config import resolve_check_user_goal_handler+from skyvern.forge.sdk.experimentation.providers import BaseExperimentationProviderfrom skyvern.forge.sdk.experimentation.slim_llm_output import get_slim_output_template_valuefrom skyvern.forge.sdk.models import Stepfrom skyvern.forge.sdk.schemas.tasks import Task@@ -164,6 +166,10 @@ FIX_TEL_INPUT_DIGIT_DROP_FLAG = "FIX_TEL_INPUT_DIGIT_DROP"COLLAPSE_SELECT_FANOUT_FLAG = "COLLAPSE_SELECT_FANOUT"COLLAPSE_CUSTOM_SELECT_FANOUT_FLAG = "COLLAPSE_CUSTOM_SELECT_FANOUT"COLLAPSE_AUTOCOMPLETE_FANOUT_FLAG = "COLLAPSE_AUTOCOMPLETE_FANOUT"+COLLAPSE_XP_ASSIGNMENT_FLAG = "COLLAPSE_XP_ASSIGNMENT"+# Nested dispatch replaces contexts, so run-stickiness is process-local and keyed by run ID.+# Cross-process re-resolution is deterministic under stable flag config.+_COLLAPSE_XP_ASSIGNMENT_MEMO: TTLCache[str, bool] = TTLCache(maxsize=100_000, ttl=86_400)DOWNLOAD_EVENT_ACTIVE_DIR_GRACE_SECONDS = 60DOWNLOAD_DUPLICATE_STEM_SUFFIX_RE = re.compile(r"(?:\s+\(\d{1,3}\)|_\d{1,3})$")@@ -1127,31 +1133,52 @@ async def _is_tel_digit_fix_enabled(task: Task) -> bool:return False-async def _is_collapse_select_fanout_enabled(task: Task) -> bool:- organization_id = task.organization_id- if not organization_id:- return False- experimentation_provider = getattr(app, "EXPERIMENTATION_PROVIDER", None)- if not experimentation_provider:- return False+async def _resolve_collapse_xp_assignment(+ experimentation_provider: BaseExperimentationProvider,+ task: Task,+ organization_id: str,+) -> bool:+ distinct_id = task.workflow_run_id or task.task_id+ if distinct_id in _COLLAPSE_XP_ASSIGNMENT_MEMO:+ return _COLLAPSE_XP_ASSIGNMENT_MEMO[distinct_id]+try:- return bool(- await experimentation_provider.is_feature_enabled_cached(- COLLAPSE_SELECT_FANOUT_FLAG,- organization_id,+ assignment = bool(+ await experimentation_provider.resolve_feature_enabled_unrecorded(+ COLLAPSE_XP_ASSIGNMENT_FLAG,+ distinct_id,properties={"organization_id": organization_id},))except Exception:- LOG.warning(- "Failed to evaluate collapse-select-fanout flag; defaulting to disabled",+ if distinct_id in _COLLAPSE_XP_ASSIGNMENT_MEMO:+ return _COLLAPSE_XP_ASSIGNMENT_MEMO[distinct_id]+ _COLLAPSE_XP_ASSIGNMENT_MEMO[distinct_id] = False+ LOG.info(+ "collapse_xp_assignment",+ workflow_run_id=task.workflow_run_id,+ task_id=task.task_id,organization_id=organization_id,- exc_info=True,+ assigned=False,+ pinned_on_error=True,)- return False+ raise++ if distinct_id in _COLLAPSE_XP_ASSIGNMENT_MEMO:+ return _COLLAPSE_XP_ASSIGNMENT_MEMO[distinct_id]+ _COLLAPSE_XP_ASSIGNMENT_MEMO[distinct_id] = assignment+ LOG.info(+ "collapse_xp_assignment",+ workflow_run_id=task.workflow_run_id,+ task_id=task.task_id,+ organization_id=organization_id,+ assigned=assignment,+ pinned_on_error=False,+ )+ return assignment-async def _is_collapse_custom_select_fanout_enabled(task: Task) -> bool:+async def _is_collapse_fanout_enabled(task: Task, family_flag: str, log_label: str) -> bool:organization_id = task.organization_idif not organization_id:return False@@ -1159,44 +1186,43 @@ async def _is_collapse_custom_select_fanout_enabled(task: Task) -> bool:if not experimentation_provider:return Falsetry:- return bool(- await experimentation_provider.is_feature_enabled_cached(- COLLAPSE_CUSTOM_SELECT_FANOUT_FLAG,- organization_id,- properties={"organization_id": organization_id},- )+ family_enabled = await experimentation_provider.is_feature_enabled_cached(+ family_flag,+ organization_id,+ properties={"organization_id": organization_id},)+ if not family_enabled:+ return False+ # PostHog hashes per flag key, so this umbrella is the only randomization source.+ # Family flags are kill switches and must never use percentage rollouts.+ return await _resolve_collapse_xp_assignment(experimentation_provider, task, organization_id)except Exception:LOG.warning(- "Failed to evaluate collapse-custom-select-fanout flag; defaulting to disabled",+ f"Failed to evaluate {log_label} flag; defaulting to disabled",organization_id=organization_id,exc_info=True,)return False+async def _is_collapse_select_fanout_enabled(task: Task) -> bool:+ return await _is_collapse_fanout_enabled(task, COLLAPSE_SELECT_FANOUT_FLAG, "collapse-select-fanout")+++async def _is_collapse_custom_select_fanout_enabled(task: Task) -> bool:+ return await _is_collapse_fanout_enabled(+ task,+ COLLAPSE_CUSTOM_SELECT_FANOUT_FLAG,+ "collapse-custom-select-fanout",+ )++async def _is_collapse_autocomplete_fanout_enabled(task: Task) -> bool:- organization_id = task.organization_id- if not organization_id:- return False- experimentation_provider = getattr(app, "EXPERIMENTATION_PROVIDER", None)- if not experimentation_provider:- return False- try:- return bool(- await experimentation_provider.is_feature_enabled_cached(- COLLAPSE_AUTOCOMPLETE_FANOUT_FLAG,- organization_id,- properties={"organization_id": organization_id},- )- )- except Exception:- LOG.warning(- "Failed to evaluate collapse-autocomplete-fanout flag; defaulting to disabled",- organization_id=organization_id,- exc_info=True,- )- return False+ return await _is_collapse_fanout_enabled(+ task,+ COLLAPSE_AUTOCOMPLETE_FANOUT_FLAG,+ "collapse-autocomplete-fanout",+ )async def verify_phone_input_digits(*, tag_name: str, locator: Locator, expected_value: str) -> None:diff --git a/tests/unit/conftest.py b/tests/unit/conftest.pyindex ddcac3a90..1c0967e24 100644--- a/tests/unit/conftest.py+++ b/tests/unit/conftest.py@@ -4,6 +4,7 @@import itertoolsimport loggingimport shutil+import sysfrom collections.abc import AsyncGenerator, Callablefrom pathlib import Pathfrom types import SimpleNamespace@@ -57,6 +58,20 @@ def setup_forge_stub_app():yield[email protected](autouse=True)+def reset_collapse_xp_assignment_memo():+ # The collapse umbrella memo is process-global by design; without clearing it,+ # an assignment memoized by one test leaks into any later test reusing the same task id.+ def _clear() -> None:+ handler_module = sys.modules.get("skyvern.webeye.actions.handler")+ if handler_module is not None:+ handler_module._COLLAPSE_XP_ASSIGNMENT_MEMO.clear()++ _clear()+ yield+ _clear()++# -- shared copilot agent-template rendering helper --_AGENT_TEMPLATE_DEFAULTS = dict(diff --git a/tests/unit/test_collapse_fanout_gates.py b/tests/unit/test_collapse_fanout_gates.pynew file mode 100644index 000000000..9073b338b--- /dev/null+++ b/tests/unit/test_collapse_fanout_gates.py@@ -0,0 +1,301 @@+import asyncio+from types import SimpleNamespace+from typing import Any++import pytest++from skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext+from skyvern.webeye.actions import handler++UMBRELLA_FLAG = "COLLAPSE_XP_ASSIGNMENT"+++class CaptureLogger:+ def __init__(self) -> None:+ self.records: list[tuple[str, str, dict[str, Any]]] = []++ def info(self, event: str, **kwargs: Any) -> None:+ self.records.append(("info", event, kwargs))++ def warning(self, event: str, **kwargs: Any) -> None:+ self.records.append(("warning", event, kwargs))+++class FakeExperimentationProvider:+ def __init__(self, results: dict[str, bool], *, raises_for: set[str] | None = None) -> None:+ self.results = results+ self.raises_for = raises_for or set()+ self.calls: list[tuple[str, str, str, dict[str, str]]] = []++ async def is_feature_enabled_cached(+ self,+ feature_name: str,+ distinct_id: str,+ properties: dict[str, str],+ ) -> bool:+ self.calls.append(("cached", feature_name, distinct_id, properties))+ if feature_name in self.raises_for:+ raise RuntimeError("feature flag unavailable")+ return self.results[feature_name]++ async def resolve_feature_enabled_unrecorded(+ self,+ feature_name: str,+ distinct_id: str,+ properties: dict[str, str],+ ) -> bool:+ self.calls.append(("unrecorded", feature_name, distinct_id, properties))+ await asyncio.sleep(0)+ if feature_name in self.raises_for:+ raise RuntimeError("feature flag unavailable")+ return self.results[feature_name]+++def _task(*, organization_id: str | None = "o_123", workflow_run_id: str | None = "wr_456") -> SimpleNamespace:+ return SimpleNamespace(+ organization_id=organization_id,+ workflow_run_id=workflow_run_id,+ task_id="tsk_789",+ )+++def _set_provider(monkeypatch: pytest.MonkeyPatch, provider: FakeExperimentationProvider) -> None:+ monkeypatch.setattr(handler.app, "EXPERIMENTATION_PROVIDER", provider)+++def _umbrella_calls(provider: FakeExperimentationProvider) -> list[tuple[str, str, str, dict[str, str]]]:+ return [call for call in provider.calls if call[1] == UMBRELLA_FLAG]+++async def test_collapse_family_off_skips_umbrella(monkeypatch: pytest.MonkeyPatch) -> None:+ provider = FakeExperimentationProvider({handler.COLLAPSE_SELECT_FANOUT_FLAG: False})+ _set_provider(monkeypatch, provider)++ assert await handler._is_collapse_select_fanout_enabled(_task()) is False+ assert provider.calls == [("cached", handler.COLLAPSE_SELECT_FANOUT_FLAG, "o_123", {"organization_id": "o_123"})]+ assert _umbrella_calls(provider) == []+++async def test_collapse_family_and_umbrella_on_uses_workflow_run_id(monkeypatch: pytest.MonkeyPatch) -> None:+ provider = FakeExperimentationProvider(+ {+ handler.COLLAPSE_SELECT_FANOUT_FLAG: True,+ UMBRELLA_FLAG: True,+ }+ )+ _set_provider(monkeypatch, provider)++ assert await handler._is_collapse_select_fanout_enabled(_task()) is True+ assert _umbrella_calls(provider) == [("unrecorded", UMBRELLA_FLAG, "wr_456", {"organization_id": "o_123"})]+++async def test_collapse_umbrella_falls_back_to_task_id(monkeypatch: pytest.MonkeyPatch) -> None:+ provider = FakeExperimentationProvider(+ {+ handler.COLLAPSE_CUSTOM_SELECT_FANOUT_FLAG: True,+ UMBRELLA_FLAG: True,+ }+ )+ _set_provider(monkeypatch, provider)++ assert await handler._is_collapse_custom_select_fanout_enabled(_task(workflow_run_id=None)) is True+ assert _umbrella_calls(provider) == [("unrecorded", UMBRELLA_FLAG, "tsk_789", {"organization_id": "o_123"})]+++async def test_collapse_umbrella_off_keeps_control_path(monkeypatch: pytest.MonkeyPatch) -> None:+ provider = FakeExperimentationProvider(+ {+ handler.COLLAPSE_AUTOCOMPLETE_FANOUT_FLAG: True,+ UMBRELLA_FLAG: False,+ }+ )+ _set_provider(monkeypatch, provider)++ assert await handler._is_collapse_autocomplete_fanout_enabled(_task()) is False+++async def test_collapse_family_error_defaults_to_disabled(monkeypatch: pytest.MonkeyPatch) -> None:+ provider = FakeExperimentationProvider(+ {+ handler.COLLAPSE_SELECT_FANOUT_FLAG: True,+ UMBRELLA_FLAG: True,+ },+ raises_for={handler.COLLAPSE_SELECT_FANOUT_FLAG},+ )+ _set_provider(monkeypatch, provider)++ assert await handler._is_collapse_select_fanout_enabled(_task()) is False+ assert _umbrella_calls(provider) == []+++async def test_collapse_missing_organization_skips_provider(monkeypatch: pytest.MonkeyPatch) -> None:+ provider = FakeExperimentationProvider({})+ _set_provider(monkeypatch, provider)++ assert await handler._is_collapse_select_fanout_enabled(_task(organization_id=None)) is False+ assert provider.calls == []+++async def test_collapse_gates_share_one_umbrella_cohort(monkeypatch: pytest.MonkeyPatch) -> None:+ provider = FakeExperimentationProvider(+ {+ handler.COLLAPSE_SELECT_FANOUT_FLAG: True,+ handler.COLLAPSE_CUSTOM_SELECT_FANOUT_FLAG: True,+ handler.COLLAPSE_AUTOCOMPLETE_FANOUT_FLAG: True,+ UMBRELLA_FLAG: True,+ }+ )+ _set_provider(monkeypatch, provider)+ task = _task()++ with skyvern_context.scoped(SkyvernContext()):+ assert await handler._is_collapse_select_fanout_enabled(task) is True+ assert await handler._is_collapse_custom_select_fanout_enabled(task) is True+ assert await handler._is_collapse_autocomplete_fanout_enabled(task) is True++ assert _umbrella_calls(provider) == [("unrecorded", UMBRELLA_FLAG, "wr_456", {"organization_id": "o_123"})]+++async def test_collapse_umbrella_assignment_stays_sticky_for_run(monkeypatch: pytest.MonkeyPatch) -> None:+ provider = FakeExperimentationProvider(+ {+ handler.COLLAPSE_SELECT_FANOUT_FLAG: True,+ handler.COLLAPSE_CUSTOM_SELECT_FANOUT_FLAG: True,+ UMBRELLA_FLAG: True,+ }+ )+ _set_provider(monkeypatch, provider)+ task = _task()++ with skyvern_context.scoped(SkyvernContext()):+ assert await handler._is_collapse_select_fanout_enabled(task) is True+ provider.results[UMBRELLA_FLAG] = False+ assert await handler._is_collapse_select_fanout_enabled(task) is True+ assert await handler._is_collapse_custom_select_fanout_enabled(task) is True++ assert _umbrella_calls(provider) == [("unrecorded", UMBRELLA_FLAG, "wr_456", {"organization_id": "o_123"})]+++async def test_collapse_umbrella_assignment_survives_context_replacement(monkeypatch: pytest.MonkeyPatch) -> None:+ provider = FakeExperimentationProvider(+ {+ handler.COLLAPSE_SELECT_FANOUT_FLAG: True,+ handler.COLLAPSE_CUSTOM_SELECT_FANOUT_FLAG: True,+ UMBRELLA_FLAG: True,+ }+ )+ _set_provider(monkeypatch, provider)+ task = _task()++ with skyvern_context.scoped(SkyvernContext(workflow_run_id=task.workflow_run_id)):+ assert await handler._is_collapse_select_fanout_enabled(task) is True+ skyvern_context.replace(SkyvernContext(workflow_run_id=task.workflow_run_id))+ provider.results[UMBRELLA_FLAG] = False+ assert await handler._is_collapse_select_fanout_enabled(task) is True+ assert await handler._is_collapse_custom_select_fanout_enabled(task) is True++ assert _umbrella_calls(provider) == [("unrecorded", UMBRELLA_FLAG, "wr_456", {"organization_id": "o_123"})]+++async def test_collapse_umbrella_never_records_against_ambient_child_context(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ provider = FakeExperimentationProvider(+ {+ handler.COLLAPSE_SELECT_FANOUT_FLAG: True,+ UMBRELLA_FLAG: True,+ }+ )+ _set_provider(monkeypatch, provider)+ logger = CaptureLogger()+ monkeypatch.setattr(handler, "LOG", logger)+ task = _task(workflow_run_id="wr_parent")++ with skyvern_context.scoped(SkyvernContext(workflow_run_id="wr_child")) as context:+ assert await handler._is_collapse_select_fanout_enabled(task) is True+ assert handler._COLLAPSE_XP_ASSIGNMENT_MEMO["wr_parent"] is True+ assert UMBRELLA_FLAG not in context.feature_flag_entries++ assert [record for record in logger.records if record[1] == "collapse_xp_assignment"] == [+ (+ "info",+ "collapse_xp_assignment",+ {+ "workflow_run_id": "wr_parent",+ "task_id": "tsk_789",+ "organization_id": "o_123",+ "assigned": True,+ "pinned_on_error": False,+ },+ )+ ]+++async def test_collapse_umbrella_first_writer_logs_once_without_lock(monkeypatch: pytest.MonkeyPatch) -> None:+ provider = FakeExperimentationProvider(+ {+ handler.COLLAPSE_SELECT_FANOUT_FLAG: True,+ UMBRELLA_FLAG: True,+ }+ )+ _set_provider(monkeypatch, provider)+ logger = CaptureLogger()+ monkeypatch.setattr(handler, "LOG", logger)+ task = _task()++ assert await asyncio.gather(+ handler._is_collapse_select_fanout_enabled(task),+ handler._is_collapse_select_fanout_enabled(task),+ ) == [True, True]+ assert len(_umbrella_calls(provider)) == 2+ assert len([record for record in logger.records if record[1] == "collapse_xp_assignment"]) == 1+++async def test_collapse_umbrella_error_pins_run_to_control(monkeypatch: pytest.MonkeyPatch) -> None:+ provider = FakeExperimentationProvider(+ {+ handler.COLLAPSE_SELECT_FANOUT_FLAG: True,+ handler.COLLAPSE_AUTOCOMPLETE_FANOUT_FLAG: True,+ UMBRELLA_FLAG: True,+ },+ raises_for={UMBRELLA_FLAG},+ )+ _set_provider(monkeypatch, provider)+ logger = CaptureLogger()+ monkeypatch.setattr(handler, "LOG", logger)+ task = _task()++ with skyvern_context.scoped(SkyvernContext(workflow_run_id=task.workflow_run_id)) as context:+ assert await handler._is_collapse_select_fanout_enabled(task) is False+ assert UMBRELLA_FLAG not in context.feature_flag_entries+ provider.raises_for.remove(UMBRELLA_FLAG)+ assert await handler._is_collapse_select_fanout_enabled(task) is False+ assert await handler._is_collapse_autocomplete_fanout_enabled(task) is False++ assert _umbrella_calls(provider) == [("unrecorded", UMBRELLA_FLAG, "wr_456", {"organization_id": "o_123"})]+ assert [record for record in logger.records if record[1] == "collapse_xp_assignment"] == [+ (+ "info",+ "collapse_xp_assignment",+ {+ "workflow_run_id": "wr_456",+ "task_id": "tsk_789",+ "organization_id": "o_123",+ "assigned": False,+ "pinned_on_error": True,+ },+ )+ ]diff --git a/tests/unit/test_no_available_option.py b/tests/unit/test_no_available_option.pyindex 890391513..c485db654 100644--- a/tests/unit/test_no_available_option.py+++ b/tests/unit/test_no_available_option.py@@ -492,7 +492,10 @@ class TestDeterministicCustomSelect:monkeypatch.setattr(handler.app,"EXPERIMENTATION_PROVIDER",- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),+ SimpleNamespace(+ is_feature_enabled_cached=AsyncMock(return_value=True),+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),+ ),)monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())get_skyvern_element = AsyncMock()@@ -520,7 +523,10 @@ class TestDeterministicCustomSelect:monkeypatch.setattr(handler.app,"EXPERIMENTATION_PROVIDER",- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=False)),+ SimpleNamespace(+ is_feature_enabled_cached=AsyncMock(return_value=False),+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=False),+ ),)get_skyvern_element = AsyncMock()get_readback_scope_element = AsyncMock()@@ -566,7 +572,10 @@ class TestDeterministicCustomSelect:monkeypatch.setattr(handler.app,"EXPERIMENTATION_PROVIDER",- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),+ SimpleNamespace(+ is_feature_enabled_cached=AsyncMock(return_value=True),+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),+ ),)monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())selected_element = _FakeCustomElement(role="listbox")@@ -593,7 +602,10 @@ class TestDeterministicCustomSelect:monkeypatch.setattr(handler.app,"EXPERIMENTATION_PROVIDER",- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),+ SimpleNamespace(+ is_feature_enabled_cached=AsyncMock(return_value=True),+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),+ ),)monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())monkeypatch.setattr(@@ -638,7 +650,10 @@ class TestDeterministicCustomSelect:monkeypatch.setattr(handler.app,"EXPERIMENTATION_PROVIDER",- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),+ SimpleNamespace(+ is_feature_enabled_cached=AsyncMock(return_value=True),+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),+ ),)monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())monkeypatch.setattr(@@ -684,7 +699,10 @@ class TestDeterministicCustomSelect:monkeypatch.setattr(handler.app,"EXPERIMENTATION_PROVIDER",- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),+ SimpleNamespace(+ is_feature_enabled_cached=AsyncMock(return_value=True),+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),+ ),)monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())monkeypatch.setattr(@@ -722,7 +740,10 @@ class TestDeterministicCustomSelect:monkeypatch.setattr(handler.app,"EXPERIMENTATION_PROVIDER",- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),+ SimpleNamespace(+ is_feature_enabled_cached=AsyncMock(return_value=True),+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),+ ),)monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())matched_states = [@@ -788,7 +809,10 @@ class TestDeterministicCustomSelect:monkeypatch.setattr(handler.app,"EXPERIMENTATION_PROVIDER",- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),+ SimpleNamespace(+ is_feature_enabled_cached=AsyncMock(return_value=True),+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),+ ),)monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())monkeypatch.setattr(@@ -828,7 +852,10 @@ class TestDeterministicCustomSelect:monkeypatch.setattr(handler.app,"EXPERIMENTATION_PROVIDER",- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),+ SimpleNamespace(+ is_feature_enabled_cached=AsyncMock(return_value=True),+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),+ ),)monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())monkeypatch.setattr(@@ -869,7 +896,10 @@ class TestDeterministicCustomSelect:monkeypatch.setattr(handler.app,"EXPERIMENTATION_PROVIDER",- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),+ SimpleNamespace(+ is_feature_enabled_cached=AsyncMock(return_value=True),+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),+ ),)monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())monkeypatch.setattr(@@ -913,7 +943,10 @@ class TestDeterministicCustomSelect:monkeypatch.setattr(handler.app,"EXPERIMENTATION_PROVIDER",- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),+ SimpleNamespace(+ is_feature_enabled_cached=AsyncMock(return_value=True),+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),+ ),)monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())monkeypatch.setattr(@@ -980,7 +1013,10 @@ class TestDeterministicCustomSelect:monkeypatch.setattr(handler.app,"EXPERIMENTATION_PROVIDER",- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),+ SimpleNamespace(+ is_feature_enabled_cached=AsyncMock(return_value=True),+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),+ ),)monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())monkeypatch.setattr(@@ -1025,7 +1061,10 @@ class TestDeterministicCustomSelect:monkeypatch.setattr(handler.app,"EXPERIMENTATION_PROVIDER",- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),+ SimpleNamespace(+ is_feature_enabled_cached=AsyncMock(return_value=True),+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),+ ),)monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())monkeypatch.setattr(@@ -1064,7 +1103,10 @@ class TestDeterministicCustomSelect:monkeypatch.setattr(handler.app,"EXPERIMENTATION_PROVIDER",- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),+ SimpleNamespace(+ is_feature_enabled_cached=AsyncMock(return_value=True),+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),+ ),)monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())monkeypatch.setattr(@@ -1195,7 +1237,10 @@ class TestDeterministicCustomSelect:monkeypatch.setattr(handler.app,"EXPERIMENTATION_PROVIDER",- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),+ SimpleNamespace(+ is_feature_enabled_cached=AsyncMock(return_value=True),+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),+ ),)monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())monkeypatch.setattr(@@ -1274,7 +1319,10 @@ class TestDeterministicCustomSelect:monkeypatch.setattr(handler.app,"EXPERIMENTATION_PROVIDER",- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),+ SimpleNamespace(+ is_feature_enabled_cached=AsyncMock(return_value=True),+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),+ ),)monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())monkeypatch.setattr(@@ -1360,7 +1408,10 @@ class TestDeterministicCustomSelect:monkeypatch.setattr(handler.app,"EXPERIMENTATION_PROVIDER",- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),+ SimpleNamespace(+ is_feature_enabled_cached=AsyncMock(return_value=True),+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),+ ),)monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())monkeypatch.setattr(diff --git a/tests/unit/test_normal_select_deterministic.py b/tests/unit/test_normal_select_deterministic.pyindex b7961e018..1c74806bc 100644--- a/tests/unit/test_normal_select_deterministic.py+++ b/tests/unit/test_normal_select_deterministic.py@@ -94,7 +94,10 @@ async def _run_normal_select(selected_index: int | None = None,llm_response: dict | None = None,) -> tuple[list, MagicMock, AsyncMock]:- provider = SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=feature_enabled))+ provider = SimpleNamespace(+ is_feature_enabled_cached=AsyncMock(return_value=feature_enabled),+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=feature_enabled),+ )normal_select_llm = AsyncMock(return_value=llm_response or {"value": "fallback", "index": None})monkeypatch.setattr(handler.app, "EXPERIMENTATION_PROVIDER", provider)