Skip to content

Automate browser based workflows with AI

fix(SKY-11571): repair main CI — add resolve_feature_enabled_unrecorded to 4 missed provider fakes (#7295)

Aaron Perez42484b20fef7
diff --git a/skyvern/forge/sdk/experimentation/providers.py b/skyvern/forge/sdk/experimentation/providers.py
index c2e0ee25b..2b2e273f7 100644
--- a/skyvern/forge/sdk/experimentation/providers.py
+++ b/skyvern/forge/sdk/experimentation/providers.py
@@ -188,6 +188,13 @@ class BaseExperimentationProvider(ABC):
)
return feature_flag_value
+ async def resolve_feature_enabled_unrecorded(
+ self, feature_name: str, distinct_id: str, properties: dict | None = None
+ ) -> bool:
+ """Resolve a flag while leaving attribution and logging to the caller."""
+ await self._prepare_feature_flag_resolution(feature_name, cached=True)
+ return await self._is_feature_enabled(feature_name, distinct_id, properties)
+
async def is_feature_enabled_cached(
self, feature_name: str, distinct_id: str, properties: dict | None = None
) -> bool:
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 1db35ef02..1fa1a947b 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -13,6 +13,7 @@ from pathlib import Path
from typing import Any, Awaitable, Callable, List, TypedDict
import structlog
+from cachetools import TTLCache
from fuzzysearch import find_near_matches
from opentelemetry import trace as otel_trace
from playwright._impl._errors import Error as PlaywrightError
@@ -84,6 +85,7 @@ from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import PendingFileChooserListener, ensure_context
from skyvern.forge.sdk.event.factory import EventStrategyFactory
from skyvern.forge.sdk.experimentation.llm_prompt_config import resolve_check_user_goal_handler
+from skyvern.forge.sdk.experimentation.providers import BaseExperimentationProvider
from skyvern.forge.sdk.experimentation.slim_llm_output import get_slim_output_template_value
from skyvern.forge.sdk.models import Step
from skyvern.forge.sdk.schemas.tasks import Task
@@ -164,6 +166,10 @@ FIX_TEL_INPUT_DIGIT_DROP_FLAG = "FIX_TEL_INPUT_DIGIT_DROP"
COLLAPSE_SELECT_FANOUT_FLAG = "COLLAPSE_SELECT_FANOUT"
COLLAPSE_CUSTOM_SELECT_FANOUT_FLAG = "COLLAPSE_CUSTOM_SELECT_FANOUT"
COLLAPSE_AUTOCOMPLETE_FANOUT_FLAG = "COLLAPSE_AUTOCOMPLETE_FANOUT"
+COLLAPSE_XP_ASSIGNMENT_FLAG = "COLLAPSE_XP_ASSIGNMENT"
+# Nested dispatch replaces contexts, so run-stickiness is process-local and keyed by run ID.
+# Cross-process re-resolution is deterministic under stable flag config.
+_COLLAPSE_XP_ASSIGNMENT_MEMO: TTLCache[str, bool] = TTLCache(maxsize=100_000, ttl=86_400)
DOWNLOAD_EVENT_ACTIVE_DIR_GRACE_SECONDS = 60
DOWNLOAD_DUPLICATE_STEM_SUFFIX_RE = re.compile(r"(?:\s+\(\d{1,3}\)|_\d{1,3})$")
@@ -1127,31 +1133,52 @@ async def _is_tel_digit_fix_enabled(task: Task) -> bool:
return False
-async def _is_collapse_select_fanout_enabled(task: Task) -> bool:
- organization_id = task.organization_id
- if not organization_id:
- return False
- experimentation_provider = getattr(app, "EXPERIMENTATION_PROVIDER", None)
- if not experimentation_provider:
- return False
+async def _resolve_collapse_xp_assignment(
+ experimentation_provider: BaseExperimentationProvider,
+ task: Task,
+ organization_id: str,
+) -> bool:
+ distinct_id = task.workflow_run_id or task.task_id
+ if distinct_id in _COLLAPSE_XP_ASSIGNMENT_MEMO:
+ return _COLLAPSE_XP_ASSIGNMENT_MEMO[distinct_id]
+
try:
- return bool(
- await experimentation_provider.is_feature_enabled_cached(
- COLLAPSE_SELECT_FANOUT_FLAG,
- organization_id,
+ assignment = bool(
+ await experimentation_provider.resolve_feature_enabled_unrecorded(
+ COLLAPSE_XP_ASSIGNMENT_FLAG,
+ distinct_id,
properties={"organization_id": organization_id},
)
)
except Exception:
- LOG.warning(
- "Failed to evaluate collapse-select-fanout flag; defaulting to disabled",
+ if distinct_id in _COLLAPSE_XP_ASSIGNMENT_MEMO:
+ return _COLLAPSE_XP_ASSIGNMENT_MEMO[distinct_id]
+ _COLLAPSE_XP_ASSIGNMENT_MEMO[distinct_id] = False
+ LOG.info(
+ "collapse_xp_assignment",
+ workflow_run_id=task.workflow_run_id,
+ task_id=task.task_id,
organization_id=organization_id,
- exc_info=True,
+ assigned=False,
+ pinned_on_error=True,
)
- return False
+ raise
+
+ if distinct_id in _COLLAPSE_XP_ASSIGNMENT_MEMO:
+ return _COLLAPSE_XP_ASSIGNMENT_MEMO[distinct_id]
+ _COLLAPSE_XP_ASSIGNMENT_MEMO[distinct_id] = assignment
+ LOG.info(
+ "collapse_xp_assignment",
+ workflow_run_id=task.workflow_run_id,
+ task_id=task.task_id,
+ organization_id=organization_id,
+ assigned=assignment,
+ pinned_on_error=False,
+ )
+ return assignment
-async def _is_collapse_custom_select_fanout_enabled(task: Task) -> bool:
+async def _is_collapse_fanout_enabled(task: Task, family_flag: str, log_label: str) -> bool:
organization_id = task.organization_id
if not organization_id:
return False
@@ -1159,44 +1186,43 @@ async def _is_collapse_custom_select_fanout_enabled(task: Task) -> bool:
if not experimentation_provider:
return False
try:
- return bool(
- await experimentation_provider.is_feature_enabled_cached(
- COLLAPSE_CUSTOM_SELECT_FANOUT_FLAG,
- organization_id,
- properties={"organization_id": organization_id},
- )
+ family_enabled = await experimentation_provider.is_feature_enabled_cached(
+ family_flag,
+ organization_id,
+ properties={"organization_id": organization_id},
)
+ if not family_enabled:
+ return False
+ # PostHog hashes per flag key, so this umbrella is the only randomization source.
+ # Family flags are kill switches and must never use percentage rollouts.
+ return await _resolve_collapse_xp_assignment(experimentation_provider, task, organization_id)
except Exception:
LOG.warning(
- "Failed to evaluate collapse-custom-select-fanout flag; defaulting to disabled",
+ f"Failed to evaluate {log_label} flag; defaulting to disabled",
organization_id=organization_id,
exc_info=True,
)
return False
+async def _is_collapse_select_fanout_enabled(task: Task) -> bool:
+ return await _is_collapse_fanout_enabled(task, COLLAPSE_SELECT_FANOUT_FLAG, "collapse-select-fanout")
+
+
+async def _is_collapse_custom_select_fanout_enabled(task: Task) -> bool:
+ return await _is_collapse_fanout_enabled(
+ task,
+ COLLAPSE_CUSTOM_SELECT_FANOUT_FLAG,
+ "collapse-custom-select-fanout",
+ )
+
+
async def _is_collapse_autocomplete_fanout_enabled(task: Task) -> bool:
- organization_id = task.organization_id
- if not organization_id:
- return False
- experimentation_provider = getattr(app, "EXPERIMENTATION_PROVIDER", None)
- if not experimentation_provider:
- return False
- try:
- return bool(
- await experimentation_provider.is_feature_enabled_cached(
- COLLAPSE_AUTOCOMPLETE_FANOUT_FLAG,
- organization_id,
- properties={"organization_id": organization_id},
- )
- )
- except Exception:
- LOG.warning(
- "Failed to evaluate collapse-autocomplete-fanout flag; defaulting to disabled",
- organization_id=organization_id,
- exc_info=True,
- )
- return False
+ return await _is_collapse_fanout_enabled(
+ task,
+ COLLAPSE_AUTOCOMPLETE_FANOUT_FLAG,
+ "collapse-autocomplete-fanout",
+ )
async def verify_phone_input_digits(*, tag_name: str, locator: Locator, expected_value: str) -> None:
diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py
index ddcac3a90..1c0967e24 100644
--- a/tests/unit/conftest.py
+++ b/tests/unit/conftest.py
@@ -4,6 +4,7 @@
import itertools
import logging
import shutil
+import sys
from collections.abc import AsyncGenerator, Callable
from pathlib import Path
from types import SimpleNamespace
@@ -57,6 +58,20 @@ def setup_forge_stub_app():
yield
[email protected](autouse=True)
+def reset_collapse_xp_assignment_memo():
+ # The collapse umbrella memo is process-global by design; without clearing it,
+ # an assignment memoized by one test leaks into any later test reusing the same task id.
+ def _clear() -> None:
+ handler_module = sys.modules.get("skyvern.webeye.actions.handler")
+ if handler_module is not None:
+ handler_module._COLLAPSE_XP_ASSIGNMENT_MEMO.clear()
+
+ _clear()
+ yield
+ _clear()
+
+
# -- shared copilot agent-template rendering helper --
_AGENT_TEMPLATE_DEFAULTS = dict(
diff --git a/tests/unit/test_collapse_fanout_gates.py b/tests/unit/test_collapse_fanout_gates.py
new file mode 100644
index 000000000..9073b338b
--- /dev/null
+++ b/tests/unit/test_collapse_fanout_gates.py
@@ -0,0 +1,301 @@
+import asyncio
+from types import SimpleNamespace
+from typing import Any
+
+import pytest
+
+from skyvern.forge.sdk.core import skyvern_context
+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
+from skyvern.webeye.actions import handler
+
+UMBRELLA_FLAG = "COLLAPSE_XP_ASSIGNMENT"
+
+
+class CaptureLogger:
+ def __init__(self) -> None:
+ self.records: list[tuple[str, str, dict[str, Any]]] = []
+
+ def info(self, event: str, **kwargs: Any) -> None:
+ self.records.append(("info", event, kwargs))
+
+ def warning(self, event: str, **kwargs: Any) -> None:
+ self.records.append(("warning", event, kwargs))
+
+
+class FakeExperimentationProvider:
+ def __init__(self, results: dict[str, bool], *, raises_for: set[str] | None = None) -> None:
+ self.results = results
+ self.raises_for = raises_for or set()
+ self.calls: list[tuple[str, str, str, dict[str, str]]] = []
+
+ async def is_feature_enabled_cached(
+ self,
+ feature_name: str,
+ distinct_id: str,
+ properties: dict[str, str],
+ ) -> bool:
+ self.calls.append(("cached", feature_name, distinct_id, properties))
+ if feature_name in self.raises_for:
+ raise RuntimeError("feature flag unavailable")
+ return self.results[feature_name]
+
+ async def resolve_feature_enabled_unrecorded(
+ self,
+ feature_name: str,
+ distinct_id: str,
+ properties: dict[str, str],
+ ) -> bool:
+ self.calls.append(("unrecorded", feature_name, distinct_id, properties))
+ await asyncio.sleep(0)
+ if feature_name in self.raises_for:
+ raise RuntimeError("feature flag unavailable")
+ return self.results[feature_name]
+
+
+def _task(*, organization_id: str | None = "o_123", workflow_run_id: str | None = "wr_456") -> SimpleNamespace:
+ return SimpleNamespace(
+ organization_id=organization_id,
+ workflow_run_id=workflow_run_id,
+ task_id="tsk_789",
+ )
+
+
+def _set_provider(monkeypatch: pytest.MonkeyPatch, provider: FakeExperimentationProvider) -> None:
+ monkeypatch.setattr(handler.app, "EXPERIMENTATION_PROVIDER", provider)
+
+
+def _umbrella_calls(provider: FakeExperimentationProvider) -> list[tuple[str, str, str, dict[str, str]]]:
+ return [call for call in provider.calls if call[1] == UMBRELLA_FLAG]
+
+
+async def test_collapse_family_off_skips_umbrella(monkeypatch: pytest.MonkeyPatch) -> None:
+ provider = FakeExperimentationProvider({handler.COLLAPSE_SELECT_FANOUT_FLAG: False})
+ _set_provider(monkeypatch, provider)
+
+ assert await handler._is_collapse_select_fanout_enabled(_task()) is False
+ assert provider.calls == [("cached", handler.COLLAPSE_SELECT_FANOUT_FLAG, "o_123", {"organization_id": "o_123"})]
+ assert _umbrella_calls(provider) == []
+
+
+async def test_collapse_family_and_umbrella_on_uses_workflow_run_id(monkeypatch: pytest.MonkeyPatch) -> None:
+ provider = FakeExperimentationProvider(
+ {
+ handler.COLLAPSE_SELECT_FANOUT_FLAG: True,
+ UMBRELLA_FLAG: True,
+ }
+ )
+ _set_provider(monkeypatch, provider)
+
+ assert await handler._is_collapse_select_fanout_enabled(_task()) is True
+ assert _umbrella_calls(provider) == [("unrecorded", UMBRELLA_FLAG, "wr_456", {"organization_id": "o_123"})]
+
+
+async def test_collapse_umbrella_falls_back_to_task_id(monkeypatch: pytest.MonkeyPatch) -> None:
+ provider = FakeExperimentationProvider(
+ {
+ handler.COLLAPSE_CUSTOM_SELECT_FANOUT_FLAG: True,
+ UMBRELLA_FLAG: True,
+ }
+ )
+ _set_provider(monkeypatch, provider)
+
+ assert await handler._is_collapse_custom_select_fanout_enabled(_task(workflow_run_id=None)) is True
+ assert _umbrella_calls(provider) == [("unrecorded", UMBRELLA_FLAG, "tsk_789", {"organization_id": "o_123"})]
+
+
+async def test_collapse_umbrella_off_keeps_control_path(monkeypatch: pytest.MonkeyPatch) -> None:
+ provider = FakeExperimentationProvider(
+ {
+ handler.COLLAPSE_AUTOCOMPLETE_FANOUT_FLAG: True,
+ UMBRELLA_FLAG: False,
+ }
+ )
+ _set_provider(monkeypatch, provider)
+
+ assert await handler._is_collapse_autocomplete_fanout_enabled(_task()) is False
+
+
+async def test_collapse_family_error_defaults_to_disabled(monkeypatch: pytest.MonkeyPatch) -> None:
+ provider = FakeExperimentationProvider(
+ {
+ handler.COLLAPSE_SELECT_FANOUT_FLAG: True,
+ UMBRELLA_FLAG: True,
+ },
+ raises_for={handler.COLLAPSE_SELECT_FANOUT_FLAG},
+ )
+ _set_provider(monkeypatch, provider)
+
+ assert await handler._is_collapse_select_fanout_enabled(_task()) is False
+ assert _umbrella_calls(provider) == []
+
+
+async def test_collapse_missing_organization_skips_provider(monkeypatch: pytest.MonkeyPatch) -> None:
+ provider = FakeExperimentationProvider({})
+ _set_provider(monkeypatch, provider)
+
+ assert await handler._is_collapse_select_fanout_enabled(_task(organization_id=None)) is False
+ assert provider.calls == []
+
+
+async def test_collapse_gates_share_one_umbrella_cohort(monkeypatch: pytest.MonkeyPatch) -> None:
+ provider = FakeExperimentationProvider(
+ {
+ handler.COLLAPSE_SELECT_FANOUT_FLAG: True,
+ handler.COLLAPSE_CUSTOM_SELECT_FANOUT_FLAG: True,
+ handler.COLLAPSE_AUTOCOMPLETE_FANOUT_FLAG: True,
+ UMBRELLA_FLAG: True,
+ }
+ )
+ _set_provider(monkeypatch, provider)
+ task = _task()
+
+ with skyvern_context.scoped(SkyvernContext()):
+ assert await handler._is_collapse_select_fanout_enabled(task) is True
+ assert await handler._is_collapse_custom_select_fanout_enabled(task) is True
+ assert await handler._is_collapse_autocomplete_fanout_enabled(task) is True
+
+ assert _umbrella_calls(provider) == [("unrecorded", UMBRELLA_FLAG, "wr_456", {"organization_id": "o_123"})]
+
+
+async def test_collapse_umbrella_assignment_stays_sticky_for_run(monkeypatch: pytest.MonkeyPatch) -> None:
+ provider = FakeExperimentationProvider(
+ {
+ handler.COLLAPSE_SELECT_FANOUT_FLAG: True,
+ handler.COLLAPSE_CUSTOM_SELECT_FANOUT_FLAG: True,
+ UMBRELLA_FLAG: True,
+ }
+ )
+ _set_provider(monkeypatch, provider)
+ task = _task()
+
+ with skyvern_context.scoped(SkyvernContext()):
+ assert await handler._is_collapse_select_fanout_enabled(task) is True
+ provider.results[UMBRELLA_FLAG] = False
+ assert await handler._is_collapse_select_fanout_enabled(task) is True
+ assert await handler._is_collapse_custom_select_fanout_enabled(task) is True
+
+ assert _umbrella_calls(provider) == [("unrecorded", UMBRELLA_FLAG, "wr_456", {"organization_id": "o_123"})]
+
+
+async def test_collapse_umbrella_assignment_survives_context_replacement(monkeypatch: pytest.MonkeyPatch) -> None:
+ provider = FakeExperimentationProvider(
+ {
+ handler.COLLAPSE_SELECT_FANOUT_FLAG: True,
+ handler.COLLAPSE_CUSTOM_SELECT_FANOUT_FLAG: True,
+ UMBRELLA_FLAG: True,
+ }
+ )
+ _set_provider(monkeypatch, provider)
+ task = _task()
+
+ with skyvern_context.scoped(SkyvernContext(workflow_run_id=task.workflow_run_id)):
+ assert await handler._is_collapse_select_fanout_enabled(task) is True
+ skyvern_context.replace(SkyvernContext(workflow_run_id=task.workflow_run_id))
+ provider.results[UMBRELLA_FLAG] = False
+ assert await handler._is_collapse_select_fanout_enabled(task) is True
+ assert await handler._is_collapse_custom_select_fanout_enabled(task) is True
+
+ assert _umbrella_calls(provider) == [("unrecorded", UMBRELLA_FLAG, "wr_456", {"organization_id": "o_123"})]
+
+
+async def test_collapse_umbrella_never_records_against_ambient_child_context(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ provider = FakeExperimentationProvider(
+ {
+ handler.COLLAPSE_SELECT_FANOUT_FLAG: True,
+ UMBRELLA_FLAG: True,
+ }
+ )
+ _set_provider(monkeypatch, provider)
+ logger = CaptureLogger()
+ monkeypatch.setattr(handler, "LOG", logger)
+ task = _task(workflow_run_id="wr_parent")
+
+ with skyvern_context.scoped(SkyvernContext(workflow_run_id="wr_child")) as context:
+ assert await handler._is_collapse_select_fanout_enabled(task) is True
+ assert handler._COLLAPSE_XP_ASSIGNMENT_MEMO["wr_parent"] is True
+ assert UMBRELLA_FLAG not in context.feature_flag_entries
+
+ assert [record for record in logger.records if record[1] == "collapse_xp_assignment"] == [
+ (
+ "info",
+ "collapse_xp_assignment",
+ {
+ "workflow_run_id": "wr_parent",
+ "task_id": "tsk_789",
+ "organization_id": "o_123",
+ "assigned": True,
+ "pinned_on_error": False,
+ },
+ )
+ ]
+
+
+async def test_collapse_umbrella_first_writer_logs_once_without_lock(monkeypatch: pytest.MonkeyPatch) -> None:
+ provider = FakeExperimentationProvider(
+ {
+ handler.COLLAPSE_SELECT_FANOUT_FLAG: True,
+ UMBRELLA_FLAG: True,
+ }
+ )
+ _set_provider(monkeypatch, provider)
+ logger = CaptureLogger()
+ monkeypatch.setattr(handler, "LOG", logger)
+ task = _task()
+
+ assert await asyncio.gather(
+ handler._is_collapse_select_fanout_enabled(task),
+ handler._is_collapse_select_fanout_enabled(task),
+ ) == [True, True]
+ assert len(_umbrella_calls(provider)) == 2
+ assert len([record for record in logger.records if record[1] == "collapse_xp_assignment"]) == 1
+
+
+async def test_collapse_umbrella_error_pins_run_to_control(monkeypatch: pytest.MonkeyPatch) -> None:
+ provider = FakeExperimentationProvider(
+ {
+ handler.COLLAPSE_SELECT_FANOUT_FLAG: True,
+ handler.COLLAPSE_AUTOCOMPLETE_FANOUT_FLAG: True,
+ UMBRELLA_FLAG: True,
+ },
+ raises_for={UMBRELLA_FLAG},
+ )
+ _set_provider(monkeypatch, provider)
+ logger = CaptureLogger()
+ monkeypatch.setattr(handler, "LOG", logger)
+ task = _task()
+
+ with skyvern_context.scoped(SkyvernContext(workflow_run_id=task.workflow_run_id)) as context:
+ assert await handler._is_collapse_select_fanout_enabled(task) is False
+ assert UMBRELLA_FLAG not in context.feature_flag_entries
+ provider.raises_for.remove(UMBRELLA_FLAG)
+ assert await handler._is_collapse_select_fanout_enabled(task) is False
+ assert await handler._is_collapse_autocomplete_fanout_enabled(task) is False
+
+ assert _umbrella_calls(provider) == [("unrecorded", UMBRELLA_FLAG, "wr_456", {"organization_id": "o_123"})]
+ assert [record for record in logger.records if record[1] == "collapse_xp_assignment"] == [
+ (
+ "info",
+ "collapse_xp_assignment",
+ {
+ "workflow_run_id": "wr_456",
+ "task_id": "tsk_789",
+ "organization_id": "o_123",
+ "assigned": False,
+ "pinned_on_error": True,
+ },
+ )
+ ]
diff --git a/tests/unit/test_no_available_option.py b/tests/unit/test_no_available_option.py
index 890391513..c485db654 100644
--- a/tests/unit/test_no_available_option.py
+++ b/tests/unit/test_no_available_option.py
@@ -492,7 +492,10 @@ class TestDeterministicCustomSelect:
monkeypatch.setattr(
handler.app,
"EXPERIMENTATION_PROVIDER",
- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),
+ SimpleNamespace(
+ is_feature_enabled_cached=AsyncMock(return_value=True),
+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),
+ ),
)
monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())
get_skyvern_element = AsyncMock()
@@ -520,7 +523,10 @@ class TestDeterministicCustomSelect:
monkeypatch.setattr(
handler.app,
"EXPERIMENTATION_PROVIDER",
- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=False)),
+ SimpleNamespace(
+ is_feature_enabled_cached=AsyncMock(return_value=False),
+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=False),
+ ),
)
get_skyvern_element = AsyncMock()
get_readback_scope_element = AsyncMock()
@@ -566,7 +572,10 @@ class TestDeterministicCustomSelect:
monkeypatch.setattr(
handler.app,
"EXPERIMENTATION_PROVIDER",
- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),
+ SimpleNamespace(
+ is_feature_enabled_cached=AsyncMock(return_value=True),
+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),
+ ),
)
monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())
selected_element = _FakeCustomElement(role="listbox")
@@ -593,7 +602,10 @@ class TestDeterministicCustomSelect:
monkeypatch.setattr(
handler.app,
"EXPERIMENTATION_PROVIDER",
- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),
+ SimpleNamespace(
+ is_feature_enabled_cached=AsyncMock(return_value=True),
+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),
+ ),
)
monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())
monkeypatch.setattr(
@@ -638,7 +650,10 @@ class TestDeterministicCustomSelect:
monkeypatch.setattr(
handler.app,
"EXPERIMENTATION_PROVIDER",
- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),
+ SimpleNamespace(
+ is_feature_enabled_cached=AsyncMock(return_value=True),
+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),
+ ),
)
monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())
monkeypatch.setattr(
@@ -684,7 +699,10 @@ class TestDeterministicCustomSelect:
monkeypatch.setattr(
handler.app,
"EXPERIMENTATION_PROVIDER",
- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),
+ SimpleNamespace(
+ is_feature_enabled_cached=AsyncMock(return_value=True),
+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),
+ ),
)
monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())
monkeypatch.setattr(
@@ -722,7 +740,10 @@ class TestDeterministicCustomSelect:
monkeypatch.setattr(
handler.app,
"EXPERIMENTATION_PROVIDER",
- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),
+ SimpleNamespace(
+ is_feature_enabled_cached=AsyncMock(return_value=True),
+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),
+ ),
)
monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())
matched_states = [
@@ -788,7 +809,10 @@ class TestDeterministicCustomSelect:
monkeypatch.setattr(
handler.app,
"EXPERIMENTATION_PROVIDER",
- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),
+ SimpleNamespace(
+ is_feature_enabled_cached=AsyncMock(return_value=True),
+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),
+ ),
)
monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())
monkeypatch.setattr(
@@ -828,7 +852,10 @@ class TestDeterministicCustomSelect:
monkeypatch.setattr(
handler.app,
"EXPERIMENTATION_PROVIDER",
- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),
+ SimpleNamespace(
+ is_feature_enabled_cached=AsyncMock(return_value=True),
+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),
+ ),
)
monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())
monkeypatch.setattr(
@@ -869,7 +896,10 @@ class TestDeterministicCustomSelect:
monkeypatch.setattr(
handler.app,
"EXPERIMENTATION_PROVIDER",
- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),
+ SimpleNamespace(
+ is_feature_enabled_cached=AsyncMock(return_value=True),
+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),
+ ),
)
monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())
monkeypatch.setattr(
@@ -913,7 +943,10 @@ class TestDeterministicCustomSelect:
monkeypatch.setattr(
handler.app,
"EXPERIMENTATION_PROVIDER",
- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),
+ SimpleNamespace(
+ is_feature_enabled_cached=AsyncMock(return_value=True),
+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),
+ ),
)
monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())
monkeypatch.setattr(
@@ -980,7 +1013,10 @@ class TestDeterministicCustomSelect:
monkeypatch.setattr(
handler.app,
"EXPERIMENTATION_PROVIDER",
- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),
+ SimpleNamespace(
+ is_feature_enabled_cached=AsyncMock(return_value=True),
+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),
+ ),
)
monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())
monkeypatch.setattr(
@@ -1025,7 +1061,10 @@ class TestDeterministicCustomSelect:
monkeypatch.setattr(
handler.app,
"EXPERIMENTATION_PROVIDER",
- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),
+ SimpleNamespace(
+ is_feature_enabled_cached=AsyncMock(return_value=True),
+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),
+ ),
)
monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())
monkeypatch.setattr(
@@ -1064,7 +1103,10 @@ class TestDeterministicCustomSelect:
monkeypatch.setattr(
handler.app,
"EXPERIMENTATION_PROVIDER",
- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),
+ SimpleNamespace(
+ is_feature_enabled_cached=AsyncMock(return_value=True),
+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),
+ ),
)
monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())
monkeypatch.setattr(
@@ -1195,7 +1237,10 @@ class TestDeterministicCustomSelect:
monkeypatch.setattr(
handler.app,
"EXPERIMENTATION_PROVIDER",
- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),
+ SimpleNamespace(
+ is_feature_enabled_cached=AsyncMock(return_value=True),
+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),
+ ),
)
monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())
monkeypatch.setattr(
@@ -1274,7 +1319,10 @@ class TestDeterministicCustomSelect:
monkeypatch.setattr(
handler.app,
"EXPERIMENTATION_PROVIDER",
- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),
+ SimpleNamespace(
+ is_feature_enabled_cached=AsyncMock(return_value=True),
+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),
+ ),
)
monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())
monkeypatch.setattr(
@@ -1360,7 +1408,10 @@ class TestDeterministicCustomSelect:
monkeypatch.setattr(
handler.app,
"EXPERIMENTATION_PROVIDER",
- SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),
+ SimpleNamespace(
+ is_feature_enabled_cached=AsyncMock(return_value=True),
+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),
+ ),
)
monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())
monkeypatch.setattr(
diff --git a/tests/unit/test_normal_select_deterministic.py b/tests/unit/test_normal_select_deterministic.py
index b7961e018..1c74806bc 100644
--- a/tests/unit/test_normal_select_deterministic.py
+++ b/tests/unit/test_normal_select_deterministic.py
@@ -94,7 +94,10 @@ async def _run_normal_select(
selected_index: int | None = None,
llm_response: dict | None = None,
) -> tuple[list, MagicMock, AsyncMock]:
- provider = SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=feature_enabled))
+ provider = SimpleNamespace(
+ is_feature_enabled_cached=AsyncMock(return_value=feature_enabled),
+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=feature_enabled),
+ )
normal_select_llm = AsyncMock(return_value=llm_response or {"value": "fallback", "index": None})
monkeypatch.setattr(handler.app, "EXPERIMENTATION_PROVIDER", provider)