:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(signals): remove jevk decision routing (#114926)
Mikayla Thompson759ef7514386
diff --git a/products/signals/backend/system_one_decision.py b/products/signals/backend/system_one_decision.pyindex fdb362d51bb7..cd1fcb711a0e 100644--- a/products/signals/backend/system_one_decision.py+++ b/products/signals/backend/system_one_decision.py@@ -4,7 +4,7 @@ import asynciofrom collections.abc import Awaitable, Callablefrom contextlib import suppressfrom time import perf_counter-from typing import TYPE_CHECKING, Generic, Literal, TypeVar, cast+from typing import Generic, Literal, TypeVarfrom uuid import uuid4from django.conf import settings@@ -29,15 +29,7 @@ from products.ml_inference.backend.facade.contracts import (NoulAnswer,)from products.ml_inference.backend.facade.enums import DecisionQuestionType-from products.signals.backend.system_one_prompts import (- JEEVES_MODEL,- JEVK_MODEL,- SystemOnePrompt,- model_experiment_prompt,-)--if TYPE_CHECKING:- from posthoganalytics.feature_flag_evaluations import FeatureFlagEvaluations+from products.signals.backend.system_one_prompts import SystemOnePromptlogger = structlog.get_logger(__name__)@@ -55,9 +47,6 @@ JEV_SHADOW_ADMISSION_TIMEOUT_SECONDS = 0.5JEV_REDIS_TIMEOUT_SECONDS = 0.1JEV_BUDGET = Budget(burst=2, per_hour=3600)JEV_TEAM_BUDGET = Budget(burst=1, per_hour=1800)-SHADOW_MODEL_FLAG = "signals-system-one-shadow-model"-SHADOW_MODEL_FLAG_TIMEOUT_SECONDS = 1.0-SHADOW_MODELS = {"jevk": JEVK_MODEL, "jeeves": JEEVES_MODEL}SAFETY_CATEGORIES = {"none": "No matching safety category",@@ -130,46 +119,6 @@ class SignalsDecision:category_confidence: float | None-@frozen-class _ShadowModelExperiment:- prompt: SystemOnePrompt- variant: Literal["jevk", "jeeves"] | None = None- status: str = "not_enrolled"- flags: "FeatureFlagEvaluations | None" = None---async def _shadow_model_experiment(team_id: int, trace_id: str, prompt: SystemOnePrompt) -> _ShadowModelExperiment:- if prompt.source != "managed" or prompt.model != JEVK_MODEL:- return _ShadowModelExperiment(prompt=prompt)- try:- async with asyncio.timeout(SHADOW_MODEL_FLAG_TIMEOUT_SECONDS):- flags = await asyncio.to_thread(- posthoganalytics.evaluate_flags,- trace_id,- groups={"project": str(team_id)},- group_properties={"project": {"id": team_id}},- flag_keys=[SHADOW_MODEL_FLAG],- )- variant = flags.get_flag(SHADOW_MODEL_FLAG)- if not isinstance(variant, str) or variant not in SHADOW_MODELS:- return _ShadowModelExperiment(prompt=prompt, flags=flags)- variant = cast(Literal["jevk", "jeeves"], variant)- payload = flags.get_flag_payload(SHADOW_MODEL_FLAG)- versions = payload.get("prompt_versions") if isinstance(payload, dict) else None- version = versions.get(prompt.name) if isinstance(versions, dict) else None- if isinstance(version, bool) or not isinstance(version, int) or version < 1:- return _ShadowModelExperiment(prompt=prompt, variant=variant, status="invalid_payload", flags=flags)- candidate = model_experiment_prompt(prompt, version, SHADOW_MODELS[variant])- if candidate is None:- return _ShadowModelExperiment(- prompt=prompt, variant=variant, status="prompt_unavailable_or_mismatched", flags=flags- )- return _ShadowModelExperiment(prompt=candidate, variant=variant, status="assigned", flags=flags)- except Exception as error:- logger.warning("Shadow model flag check failed", stage="model_selection", error_type=type(error).__name__)- return _ShadowModelExperiment(prompt=prompt, status=type(error).__name__)--async def model_mode(team_id: int) -> ModelMode:try:value = await asyncio.to_thread(@@ -316,7 +265,6 @@ async def run_model_decision(return await traditional(None)trace_id = str(uuid4())- experiment = _ShadowModelExperiment(prompt=prompt)async def run_traditional() -> _SignalsModelCallResult[T]:started = perf_counter()@@ -330,12 +278,8 @@ async def run_model_decision(return _SignalsModelCallResult(value=None, error=error, latency_seconds=perf_counter() - started)async def run_system_one() -> _SignalsModelCallResult[SignalsDecision]:- nonlocal experiment, promptstarted = perf_counter()try:- if mode == "system-one-shadow":- experiment = await _shadow_model_experiment(team_id, trace_id, prompt)- prompt = experiment.promptresult = await _query(team_id, stage, state, prompt, trace_id, source_id, source_product, mode)return _SignalsModelCallResult(value=result, error=None, latency_seconds=perf_counter() - started)except _JevAdmissionError as error:@@ -437,8 +381,6 @@ async def run_model_decision("$ai_prompt_name": prompt.name,"$ai_prompt_version": str(prompt.version) if prompt.version is not None else None,"system_one_prompt_source": prompt.source,- "system_one_model_experiment_variant": experiment.variant,- "system_one_model_experiment_status": experiment.status,"system_one_requested_model": prompt.model,}if system_one is not None:@@ -480,7 +422,6 @@ async def run_model_decision(event="signals_typesafe_decision_evaluated",distinct_id=f"team-{team_id}",properties=properties,- flags=experiment.flags,)if decision_error is not None:if mode == "system-one-only":diff --git a/products/signals/backend/system_one_prompts.py b/products/signals/backend/system_one_prompts.pyindex 4ce136c4f810..0f1c947dad10 100644--- a/products/signals/backend/system_one_prompts.py+++ b/products/signals/backend/system_one_prompts.py@@ -15,10 +15,9 @@ logger = structlog.get_logger(__name__)PROMPT_LABEL = "production"PROMPT_REFRESH_SECONDS = 60-JEVK_MODEL = "posthog/hogference/jevk5-fp8-0.2"JEEVES_MODEL = "posthog/hogference/jeeves-0.1"DEFAULT_SYSTEM_ONE_MODEL = JEEVES_MODEL-ALLOWED_SYSTEM_ONE_MODELS = {JEVK_MODEL, JEEVES_MODEL}+ALLOWED_SYSTEM_ONE_MODELS = {JEEVES_MODEL}SAFETY_RESPONSE_FIELDS = {"signals-signal-safety-system-one": ("safe", "threat_type", "explanation"),"signals-report-safety-system-one": ("choice", "explanation"),@@ -172,20 +171,3 @@ _CACHE = _PromptCache()def current_prompt(fallback: SystemOnePrompt, *, version: int | None = None) -> SystemOnePrompt:return _CACHE.current(fallback, version=version)---def model_experiment_prompt(primary: SystemOnePrompt, version: int, model: str) -> SystemOnePrompt | None:- if primary.source == "managed" and primary.version == version and primary.model == model:- return primary- fallback = bundled_prompt(primary.name, primary.policy, primary.question, primary.threshold)- candidate = current_prompt(fallback, version=version)- if (- candidate.source != "managed"- or candidate.version != version- or candidate.model != model- or candidate.policy != primary.policy- or candidate.question != primary.question- or candidate.threshold != primary.threshold- ):- return None- return candidatediff --git a/products/signals/backend/test/test_system_one_decision.py b/products/signals/backend/test/test_system_one_decision.pyindex 7af362699d81..c98a4747e2f5 100644--- a/products/signals/backend/test/test_system_one_decision.py+++ b/products/signals/backend/test/test_system_one_decision.py@@ -27,7 +27,6 @@ from products.signals.backend.emission.pipeline import filter_actionablefrom products.signals.backend.emission.registry import SignalEmitterOutputfrom products.signals.backend.system_one_decision import (JEV_TIMEOUT_SECONDS,- SHADOW_MODEL_FLAG,ModelMode,SignalsDecision,SignalsDecisionError,@@ -38,7 +37,6 @@ from products.signals.backend.system_one_decision import (from products.signals.backend.system_one_prompts import (DEFAULT_SYSTEM_ONE_MODEL,JEEVES_MODEL,- JEVK_MODEL,SystemOnePrompt,bundled_prompt,)@@ -200,7 +198,6 @@ async def test_shadow_disagreement_keeps_primary_result_and_records_usage() -> Nassert properties["system_one_requested_model"] == JEEVES_MODELassert properties["system_one_model"] == JEEVES_MODELassert properties["system_one_prompt_source"] == "bundled"- assert properties["system_one_model_experiment_status"] == "not_enrolled"assert properties["system_one_estimated_cost_usd"] is Noneassert decide.call_count == 1assert decide.call_args.args[0].model == JEEVES_MODEL@@ -214,93 +211,43 @@ async def test_shadow_disagreement_keeps_primary_result_and_records_usage() -> N@pytest.mark.asyncio@pytest.mark.parametrize("stage", ["actionability", "signal_safety", "report_safety"])[email protected]("variant", ["jevk", "jeeves"])-async def test_shadow_model_split_calls_one_model_and_keeps_sonnet_deciding(stage: str, variant: str) -> None:+async def test_managed_shadow_calls_jeeves_once_and_keeps_traditional_deciding(stage: str) -> None:prompt = replace(bundled_prompt("signals-actionability-issue", "policy", "question", 0.9),- model=JEVK_MODEL,source="managed",version=2,)- model = JEVK_MODEL if variant == "jevk" else JEEVES_MODEL- selected = replace(prompt, model=model, version=2 if variant == "jevk" else 3)- result = DecisionResult(model=model, answers={"actionable": NoulAnswer(probability=0.99)}, input_tokens=100)+ result = DecisionResult(model=JEEVES_MODEL, answers={"actionable": NoulAnswer(probability=0.99)}, input_tokens=100)if stage != "actionability":result = DecisionResult(- model=model,+ model=JEEVES_MODEL,answers={"safe": NoulAnswer(probability=0.99),"category": ChoiceAnswer(choice="none", confidence=1.0, probabilities={"none": 1.0}),},input_tokens=100,)- flags = MagicMock()- flags.get_flag.return_value = variant- flags.get_flag_payload.return_value = {"prompt_versions": {prompt.name: selected.version}}- sonnet = AsyncMock(return_value=False)+ traditional = AsyncMock(return_value=False)with (patch("products.signals.backend.system_one_decision.posthoganalytics.get_feature_flag",return_value="typesafe-shadow",),- patch(- "products.signals.backend.system_one_decision.posthoganalytics.evaluate_flags", return_value=flags- ) as evaluate,- patch("products.signals.backend.system_one_decision.model_experiment_prompt", return_value=selected),+ patch("products.signals.backend.system_one_decision.posthoganalytics.evaluate_flags") as evaluate,patch("products.signals.backend.system_one_decision.posthoganalytics.capture") as capture,patch("products.signals.backend.system_one_decision.decision_api.decide_when_available", return_value=result) as decide,):- assert await _run_actionability(stage=stage, prompt=prompt, traditional=sonnet) is False- sonnet.assert_awaited_once()+ assert await _run_actionability(stage=stage, prompt=prompt, traditional=traditional) is False+ traditional.assert_awaited_once()assert decide.call_count == 1- assert decide.call_args.args[0].model == model+ assert decide.call_args.args[0].model == JEEVES_MODELproperties = capture.call_args.kwargs["properties"]assert properties["system_one_deciding_provider"] == "traditional"- assert properties["system_one_model_experiment_variant"] == variant- assert properties["system_one_model_experiment_status"] == "assigned"- assert properties["system_one_model"] == model+ assert properties["system_one_model"] == JEEVES_MODELassert properties["disagreement"] is True- assert evaluate.call_args.args == (properties["signals_decision_id"],)- flags.get_flag.assert_called_once_with(SHADOW_MODEL_FLAG)--[email protected]("outcome", ["disabled", "invalid_payload", "prompt_unavailable", "flag_error"])-async def test_unavailable_shadow_experiment_falls_back_to_one_jevk_call(outcome: str) -> None:- prompt = replace(- bundled_prompt("signals-actionability-issue", "policy", "question", 0.9),- model=JEVK_MODEL,- source="managed",- version=2,- )- flags = MagicMock()- flags.get_flag.return_value = False if outcome == "disabled" else "jeeves"- flags.get_flag_payload.return_value = {- "prompt_versions": {prompt.name: True if outcome == "invalid_payload" else 3}- }- with (- patch(- "products.signals.backend.system_one_decision.posthoganalytics.get_feature_flag",- return_value="typesafe-shadow",- ),- patch(- "products.signals.backend.system_one_decision.posthoganalytics.evaluate_flags",- side_effect=RuntimeError("secret") if outcome == "flag_error" else None,- return_value=flags,- ),- patch("products.signals.backend.system_one_decision.model_experiment_prompt", return_value=None),- patch("products.signals.backend.system_one_decision.posthoganalytics.capture") as capture,- patch(- "products.signals.backend.system_one_decision.decision_api.decide_when_available",- return_value=_actionability_result(),- ) as decide,- ):- assert await _run_actionability(prompt=prompt, traditional=AsyncMock(return_value=False)) is False- assert decide.call_count == 1- assert decide.call_args.args[0].model == JEVK_MODEL- assert capture.call_args.kwargs["properties"]["system_one_model_experiment_status"] != "assigned"+ evaluate.assert_not_called()@pytest.mark.asynciodiff --git a/products/signals/backend/test/test_system_one_prompts.py b/products/signals/backend/test/test_system_one_prompts.pyindex f84812b681d7..8fdf47d0d97e 100644--- a/products/signals/backend/test/test_system_one_prompts.py+++ b/products/signals/backend/test/test_system_one_prompts.py@@ -8,21 +8,18 @@ from posthoganalytics.ai.prompts import PromptResultfrom products.signals.backend.system_one_prompts import (DEFAULT_SYSTEM_ONE_MODEL,- JEVK_MODEL,SystemOnePrompt,_parse_prompt,_PromptCache,_PromptState,bundled_prompt,fetch_prompt,- model_experiment_prompt,)from products.signals.backend.temporal.report_safety_judge import REPORT_SAFETY_SYSTEM_ONE_PROMPTfrom products.signals.backend.temporal.safety_filter import SIGNAL_SAFETY_SYSTEM_ONE_PROMPT[email protected]("model", [JEVK_MODEL, DEFAULT_SYSTEM_ONE_MODEL])-def test_managed_prompt_keeps_policy_model_question_and_threshold_together(model: str) -> None:+def test_managed_prompt_keeps_policy_model_question_and_threshold_together() -> None:fallback = bundled_prompt("signals-actionability-issue", "old {description}", "old question", 0.85)policy = ("When in doubt, classify as ACTIONABLE.\n"@@ -34,7 +31,7 @@ def test_managed_prompt_keeps_policy_model_question_and_threshold_together(modelname=fallback.name,version=2,prompt=policy,- config={"model": model, "question": "new question", "threshold": 0.91},+ config={"model": DEFAULT_SYSTEM_ONE_MODEL, "question": "new question", "threshold": 0.91},)with (@@ -47,7 +44,7 @@ def test_managed_prompt_keeps_policy_model_question_and_threshold_together(modelassert managed is not Noneassert (managed.policy, managed.model, managed.question, managed.threshold, managed.version, managed.source) == (policy,- model,+ DEFAULT_SYSTEM_ONE_MODEL,"new question",0.91,2,@@ -185,28 +182,6 @@ def test_failed_refresh_reverts_a_managed_safety_prompt_to_bundled() -> None:cache._executor.shutdown(wait=True)[email protected]("changed", [None, "policy", "question", "threshold", "model", "version", "source"])-def test_model_experiment_requires_a_model_only_candidate(changed: str | None) -> None:- primary = replace(SIGNAL_SAFETY_SYSTEM_ONE_PROMPT, model=JEVK_MODEL, source="managed", version=2)- candidate = replace(primary, model="posthog/hogference/jeeves-0.1", version=3)- if changed == "policy":- candidate = replace(candidate, policy="different policy")- elif changed == "question":- candidate = replace(candidate, question="different question")- elif changed == "threshold":- candidate = replace(candidate, threshold=0.5)- elif changed == "model":- candidate = replace(candidate, model=primary.model)- elif changed == "version":- candidate = replace(candidate, version=4)- elif changed == "source":- candidate = replace(candidate, source="bundled")- with patch("products.signals.backend.system_one_prompts.current_prompt", return_value=candidate):- assert model_experiment_prompt(primary, 3, "posthog/hogference/jeeves-0.1") == (- candidate if changed is None else None- )--def test_versioned_shadow_refresh_does_not_replace_production() -> None:fallback = SIGNAL_SAFETY_SYSTEM_ONE_PROMPTprimary = replace(fallback, source="managed", version=2)@@ -222,3 +197,25 @@ def test_versioned_shadow_refresh_does_not_replace_production() -> None:assert cache.current(fallback) == primaryfinally:cache._executor.shutdown(wait=True)++[email protected]("fallback", [SIGNAL_SAFETY_SYSTEM_ONE_PROMPT, REPORT_SAFETY_SYSTEM_ONE_PROMPT])+def test_managed_jevk_prompt_is_rejected(fallback: SystemOnePrompt) -> None:+ result = PromptResult(+ source="api",+ name=fallback.name,+ version=1,+ prompt=fallback.policy,+ config={+ "model": "posthog/hogference/jevk5-fp8-0.2",+ "question": fallback.question,+ "threshold": fallback.threshold,+ },+ )+ with (+ patch("products.signals.backend.system_one_prompts.posthoganalytics.personal_api_key", "test-key"),+ patch("products.signals.backend.system_one_prompts.Prompts") as prompts,+ ):+ prompts.return_value.get.return_value = result+ assert fetch_prompt(fallback) is None+ assert fallback.model == DEFAULT_SYSTEM_ONE_MODEL