:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
from types import SimpleNamespace
from unittest.mock import AsyncMock, patch
from django.core.cache import cache
from django.test import SimpleTestCase, override_settings
from posthog.llm.system_one import NoulAnswer, Question, SystemOneNotConfigured, SystemOneRequestFailed, SystemOneResult
from posthog.llm.system_one_client import GatewaySystemOneClient
from posthog.tasks.usage_report import POSTHOG_AI_PRODUCTS
from products.today.backend.facade.contracts import JevTimedOut
from products.today.backend.logic.jev import GatewayJev
LOCAL_CACHE = {"default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"}}
CLIENT = GatewaySystemOneClient(
url="https://gateway.example.com", api_key="test-key", headers={}, model="jev", timeout=30
)
def answered(state: dict[str, str], questions: dict[str, Question]) -> SystemOneResult:
return SystemOneResult(model="jev", answers={key: NoulAnswer(probability=0.8) for key in questions}, input_tokens=1)
@override_settings(CACHES=LOCAL_CACHE)
class TestGatewayJev(SimpleTestCase):
def setUp(self) -> None:
cache.clear()
def _ask(self, adecide: AsyncMock, items: list[str], team_id: int = 1) -> list[float | None]:
with (
patch("products.today.backend.logic.jev.build_system_one_client", return_value=CLIENT) as build,
patch.object(GatewaySystemOneClient, "adecide", adecide),
):
answers = GatewayJev(team_id=team_id, distinct_id="person").yes_probability(items, "Is it broken?")
assert all(call.kwargs["ai_product"] not in POSTHOG_AI_PRODUCTS for call in build.call_args_list)
return answers
def test_asks_the_model_once_for_each_distinct_text(self) -> None:
adecide = AsyncMock(side_effect=answered)
first = self._ask(adecide, ["cart", "spinner", "cart"])
second = self._ask(adecide, ["spinner", "cart"])
other_team = self._ask(adecide, ["cart"], team_id=2)
assert (first, second, other_team) == ([0.8, 0.8, 0.8], [0.8, 0.8], [0.8])
assert [sorted(call.kwargs["state"].values()) for call in adecide.await_args_list] == [
["cart", "spinner"],
["cart"],
]
def test_keeps_the_answers_of_batches_that_succeed_when_another_fails(self) -> None:
items = [f"text {index}" for index in range(40)]
def first_batch_fails(state: dict[str, str], questions: dict[str, Question]) -> SystemOneResult:
if "text 0" in state.values():
raise SystemOneRequestFailed("gateway down", status_code=500)
return answered(state, questions)
with self.assertRaises(SystemOneRequestFailed):
self._ask(AsyncMock(side_effect=first_batch_fails), items)
retry = AsyncMock(side_effect=answered)
assert self._ask(retry, items) == [0.8] * 40
assert [len(call.kwargs["state"]) for call in retry.await_args_list] == [32]
def test_stops_asking_once_the_request_deadline_passes(self) -> None:
clock = SimpleNamespace(now=0.0)
with (
patch("products.today.backend.logic.jev.build_system_one_client", return_value=CLIENT),
patch.object(GatewaySystemOneClient, "adecide", AsyncMock(side_effect=answered)),
patch("products.today.backend.logic.jev.time", SimpleNamespace(monotonic=lambda: clock.now)),
):
jev = GatewayJev(team_id=1, distinct_id="person")
jev.yes_probability(["cart"], "Is it broken?")
clock.now = 60.0
with self.assertRaises(JevTimedOut):
jev.yes_probability(["spinner"], "Is it broken?")
def test_answers_from_the_cache_without_a_gateway(self) -> None:
self._ask(AsyncMock(side_effect=answered), ["cart"])
with patch(
"products.today.backend.logic.jev.build_system_one_client", side_effect=SystemOneNotConfigured("no gateway")
):
assert GatewayJev(team_id=1, distinct_id="person").yes_probability(["cart"], "Is it broken?") == [0.8]