Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(signals): remove jevk decision routing (#114926)

Mikayla Thompson759ef7514386
diff --git a/products/signals/backend/system_one_decision.py b/products/signals/backend/system_one_decision.py
index fdb362d51bb7..cd1fcb711a0e 100644
--- a/products/signals/backend/system_one_decision.py
+++ b/products/signals/backend/system_one_decision.py
@@ -4,7 +4,7 @@ import asyncio
from collections.abc import Awaitable, Callable
from contextlib import suppress
from time import perf_counter
-from typing import TYPE_CHECKING, Generic, Literal, TypeVar, cast
+from typing import Generic, Literal, TypeVar
from uuid import uuid4
from django.conf import settings
@@ -29,15 +29,7 @@ from products.ml_inference.backend.facade.contracts import (
NoulAnswer,
)
from products.ml_inference.backend.facade.enums import DecisionQuestionType
-from products.signals.backend.system_one_prompts import (
- JEEVES_MODEL,
- JEVK_MODEL,
- SystemOnePrompt,
- model_experiment_prompt,
-)
-
-if TYPE_CHECKING:
- from posthoganalytics.feature_flag_evaluations import FeatureFlagEvaluations
+from products.signals.backend.system_one_prompts import SystemOnePrompt
logger = structlog.get_logger(__name__)
@@ -55,9 +47,6 @@ JEV_SHADOW_ADMISSION_TIMEOUT_SECONDS = 0.5
JEV_REDIS_TIMEOUT_SECONDS = 0.1
JEV_BUDGET = Budget(burst=2, per_hour=3600)
JEV_TEAM_BUDGET = Budget(burst=1, per_hour=1800)
-SHADOW_MODEL_FLAG = "signals-system-one-shadow-model"
-SHADOW_MODEL_FLAG_TIMEOUT_SECONDS = 1.0
-SHADOW_MODELS = {"jevk": JEVK_MODEL, "jeeves": JEEVES_MODEL}
SAFETY_CATEGORIES = {
"none": "No matching safety category",
@@ -130,46 +119,6 @@ class SignalsDecision:
category_confidence: float | None
-@frozen
-class _ShadowModelExperiment:
- prompt: SystemOnePrompt
- variant: Literal["jevk", "jeeves"] | None = None
- status: str = "not_enrolled"
- flags: "FeatureFlagEvaluations | None" = None
-
-
-async def _shadow_model_experiment(team_id: int, trace_id: str, prompt: SystemOnePrompt) -> _ShadowModelExperiment:
- if prompt.source != "managed" or prompt.model != JEVK_MODEL:
- return _ShadowModelExperiment(prompt=prompt)
- try:
- async with asyncio.timeout(SHADOW_MODEL_FLAG_TIMEOUT_SECONDS):
- flags = await asyncio.to_thread(
- posthoganalytics.evaluate_flags,
- trace_id,
- groups={"project": str(team_id)},
- group_properties={"project": {"id": team_id}},
- flag_keys=[SHADOW_MODEL_FLAG],
- )
- variant = flags.get_flag(SHADOW_MODEL_FLAG)
- if not isinstance(variant, str) or variant not in SHADOW_MODELS:
- return _ShadowModelExperiment(prompt=prompt, flags=flags)
- variant = cast(Literal["jevk", "jeeves"], variant)
- payload = flags.get_flag_payload(SHADOW_MODEL_FLAG)
- versions = payload.get("prompt_versions") if isinstance(payload, dict) else None
- version = versions.get(prompt.name) if isinstance(versions, dict) else None
- if isinstance(version, bool) or not isinstance(version, int) or version < 1:
- return _ShadowModelExperiment(prompt=prompt, variant=variant, status="invalid_payload", flags=flags)
- candidate = model_experiment_prompt(prompt, version, SHADOW_MODELS[variant])
- if candidate is None:
- return _ShadowModelExperiment(
- prompt=prompt, variant=variant, status="prompt_unavailable_or_mismatched", flags=flags
- )
- return _ShadowModelExperiment(prompt=candidate, variant=variant, status="assigned", flags=flags)
- except Exception as error:
- logger.warning("Shadow model flag check failed", stage="model_selection", error_type=type(error).__name__)
- return _ShadowModelExperiment(prompt=prompt, status=type(error).__name__)
-
-
async def model_mode(team_id: int) -> ModelMode:
try:
value = await asyncio.to_thread(
@@ -316,7 +265,6 @@ async def run_model_decision(
return await traditional(None)
trace_id = str(uuid4())
- experiment = _ShadowModelExperiment(prompt=prompt)
async def run_traditional() -> _SignalsModelCallResult[T]:
started = perf_counter()
@@ -330,12 +278,8 @@ async def run_model_decision(
return _SignalsModelCallResult(value=None, error=error, latency_seconds=perf_counter() - started)
async def run_system_one() -> _SignalsModelCallResult[SignalsDecision]:
- nonlocal experiment, prompt
started = perf_counter()
try:
- if mode == "system-one-shadow":
- experiment = await _shadow_model_experiment(team_id, trace_id, prompt)
- prompt = experiment.prompt
result = await _query(team_id, stage, state, prompt, trace_id, source_id, source_product, mode)
return _SignalsModelCallResult(value=result, error=None, latency_seconds=perf_counter() - started)
except _JevAdmissionError as error:
@@ -437,8 +381,6 @@ async def run_model_decision(
"$ai_prompt_name": prompt.name,
"$ai_prompt_version": str(prompt.version) if prompt.version is not None else None,
"system_one_prompt_source": prompt.source,
- "system_one_model_experiment_variant": experiment.variant,
- "system_one_model_experiment_status": experiment.status,
"system_one_requested_model": prompt.model,
}
if system_one is not None:
@@ -480,7 +422,6 @@ async def run_model_decision(
event="signals_typesafe_decision_evaluated",
distinct_id=f"team-{team_id}",
properties=properties,
- flags=experiment.flags,
)
if decision_error is not None:
if mode == "system-one-only":
diff --git a/products/signals/backend/system_one_prompts.py b/products/signals/backend/system_one_prompts.py
index 4ce136c4f810..0f1c947dad10 100644
--- a/products/signals/backend/system_one_prompts.py
+++ b/products/signals/backend/system_one_prompts.py
@@ -15,10 +15,9 @@ logger = structlog.get_logger(__name__)
PROMPT_LABEL = "production"
PROMPT_REFRESH_SECONDS = 60
-JEVK_MODEL = "posthog/hogference/jevk5-fp8-0.2"
JEEVES_MODEL = "posthog/hogference/jeeves-0.1"
DEFAULT_SYSTEM_ONE_MODEL = JEEVES_MODEL
-ALLOWED_SYSTEM_ONE_MODELS = {JEVK_MODEL, JEEVES_MODEL}
+ALLOWED_SYSTEM_ONE_MODELS = {JEEVES_MODEL}
SAFETY_RESPONSE_FIELDS = {
"signals-signal-safety-system-one": ("safe", "threat_type", "explanation"),
"signals-report-safety-system-one": ("choice", "explanation"),
@@ -172,20 +171,3 @@ _CACHE = _PromptCache()
def current_prompt(fallback: SystemOnePrompt, *, version: int | None = None) -> SystemOnePrompt:
return _CACHE.current(fallback, version=version)
-
-
-def model_experiment_prompt(primary: SystemOnePrompt, version: int, model: str) -> SystemOnePrompt | None:
- if primary.source == "managed" and primary.version == version and primary.model == model:
- return primary
- fallback = bundled_prompt(primary.name, primary.policy, primary.question, primary.threshold)
- candidate = current_prompt(fallback, version=version)
- if (
- candidate.source != "managed"
- or candidate.version != version
- or candidate.model != model
- or candidate.policy != primary.policy
- or candidate.question != primary.question
- or candidate.threshold != primary.threshold
- ):
- return None
- return candidate
diff --git a/products/signals/backend/test/test_system_one_decision.py b/products/signals/backend/test/test_system_one_decision.py
index 7af362699d81..c98a4747e2f5 100644
--- a/products/signals/backend/test/test_system_one_decision.py
+++ b/products/signals/backend/test/test_system_one_decision.py
@@ -27,7 +27,6 @@ from products.signals.backend.emission.pipeline import filter_actionable
from products.signals.backend.emission.registry import SignalEmitterOutput
from products.signals.backend.system_one_decision import (
JEV_TIMEOUT_SECONDS,
- SHADOW_MODEL_FLAG,
ModelMode,
SignalsDecision,
SignalsDecisionError,
@@ -38,7 +37,6 @@ from products.signals.backend.system_one_decision import (
from products.signals.backend.system_one_prompts import (
DEFAULT_SYSTEM_ONE_MODEL,
JEEVES_MODEL,
- JEVK_MODEL,
SystemOnePrompt,
bundled_prompt,
)
@@ -200,7 +198,6 @@ async def test_shadow_disagreement_keeps_primary_result_and_records_usage() -> N
assert properties["system_one_requested_model"] == JEEVES_MODEL
assert properties["system_one_model"] == JEEVES_MODEL
assert properties["system_one_prompt_source"] == "bundled"
- assert properties["system_one_model_experiment_status"] == "not_enrolled"
assert properties["system_one_estimated_cost_usd"] is None
assert decide.call_count == 1
assert decide.call_args.args[0].model == JEEVES_MODEL
@@ -214,93 +211,43 @@ async def test_shadow_disagreement_keeps_primary_result_and_records_usage() -> N
@pytest.mark.asyncio
@pytest.mark.parametrize("stage", ["actionability", "signal_safety", "report_safety"])
[email protected]("variant", ["jevk", "jeeves"])
-async def test_shadow_model_split_calls_one_model_and_keeps_sonnet_deciding(stage: str, variant: str) -> None:
+async def test_managed_shadow_calls_jeeves_once_and_keeps_traditional_deciding(stage: str) -> None:
prompt = replace(
bundled_prompt("signals-actionability-issue", "policy", "question", 0.9),
- model=JEVK_MODEL,
source="managed",
version=2,
)
- model = JEVK_MODEL if variant == "jevk" else JEEVES_MODEL
- selected = replace(prompt, model=model, version=2 if variant == "jevk" else 3)
- result = DecisionResult(model=model, answers={"actionable": NoulAnswer(probability=0.99)}, input_tokens=100)
+ result = DecisionResult(model=JEEVES_MODEL, answers={"actionable": NoulAnswer(probability=0.99)}, input_tokens=100)
if stage != "actionability":
result = DecisionResult(
- model=model,
+ model=JEEVES_MODEL,
answers={
"safe": NoulAnswer(probability=0.99),
"category": ChoiceAnswer(choice="none", confidence=1.0, probabilities={"none": 1.0}),
},
input_tokens=100,
)
- flags = MagicMock()
- flags.get_flag.return_value = variant
- flags.get_flag_payload.return_value = {"prompt_versions": {prompt.name: selected.version}}
- sonnet = AsyncMock(return_value=False)
+ traditional = AsyncMock(return_value=False)
with (
patch(
"products.signals.backend.system_one_decision.posthoganalytics.get_feature_flag",
return_value="typesafe-shadow",
),
- patch(
- "products.signals.backend.system_one_decision.posthoganalytics.evaluate_flags", return_value=flags
- ) as evaluate,
- patch("products.signals.backend.system_one_decision.model_experiment_prompt", return_value=selected),
+ patch("products.signals.backend.system_one_decision.posthoganalytics.evaluate_flags") as evaluate,
patch("products.signals.backend.system_one_decision.posthoganalytics.capture") as capture,
patch(
"products.signals.backend.system_one_decision.decision_api.decide_when_available", return_value=result
) as decide,
):
- assert await _run_actionability(stage=stage, prompt=prompt, traditional=sonnet) is False
- sonnet.assert_awaited_once()
+ assert await _run_actionability(stage=stage, prompt=prompt, traditional=traditional) is False
+ traditional.assert_awaited_once()
assert decide.call_count == 1
- assert decide.call_args.args[0].model == model
+ assert decide.call_args.args[0].model == JEEVES_MODEL
properties = capture.call_args.kwargs["properties"]
assert properties["system_one_deciding_provider"] == "traditional"
- assert properties["system_one_model_experiment_variant"] == variant
- assert properties["system_one_model_experiment_status"] == "assigned"
- assert properties["system_one_model"] == model
+ assert properties["system_one_model"] == JEEVES_MODEL
assert properties["disagreement"] is True
- assert evaluate.call_args.args == (properties["signals_decision_id"],)
- flags.get_flag.assert_called_once_with(SHADOW_MODEL_FLAG)
-
-
[email protected]("outcome", ["disabled", "invalid_payload", "prompt_unavailable", "flag_error"])
-async def test_unavailable_shadow_experiment_falls_back_to_one_jevk_call(outcome: str) -> None:
- prompt = replace(
- bundled_prompt("signals-actionability-issue", "policy", "question", 0.9),
- model=JEVK_MODEL,
- source="managed",
- version=2,
- )
- flags = MagicMock()
- flags.get_flag.return_value = False if outcome == "disabled" else "jeeves"
- flags.get_flag_payload.return_value = {
- "prompt_versions": {prompt.name: True if outcome == "invalid_payload" else 3}
- }
- with (
- patch(
- "products.signals.backend.system_one_decision.posthoganalytics.get_feature_flag",
- return_value="typesafe-shadow",
- ),
- patch(
- "products.signals.backend.system_one_decision.posthoganalytics.evaluate_flags",
- side_effect=RuntimeError("secret") if outcome == "flag_error" else None,
- return_value=flags,
- ),
- patch("products.signals.backend.system_one_decision.model_experiment_prompt", return_value=None),
- patch("products.signals.backend.system_one_decision.posthoganalytics.capture") as capture,
- patch(
- "products.signals.backend.system_one_decision.decision_api.decide_when_available",
- return_value=_actionability_result(),
- ) as decide,
- ):
- assert await _run_actionability(prompt=prompt, traditional=AsyncMock(return_value=False)) is False
- assert decide.call_count == 1
- assert decide.call_args.args[0].model == JEVK_MODEL
- assert capture.call_args.kwargs["properties"]["system_one_model_experiment_status"] != "assigned"
+ evaluate.assert_not_called()
@pytest.mark.asyncio
diff --git a/products/signals/backend/test/test_system_one_prompts.py b/products/signals/backend/test/test_system_one_prompts.py
index f84812b681d7..8fdf47d0d97e 100644
--- a/products/signals/backend/test/test_system_one_prompts.py
+++ b/products/signals/backend/test/test_system_one_prompts.py
@@ -8,21 +8,18 @@ from posthoganalytics.ai.prompts import PromptResult
from products.signals.backend.system_one_prompts import (
DEFAULT_SYSTEM_ONE_MODEL,
- JEVK_MODEL,
SystemOnePrompt,
_parse_prompt,
_PromptCache,
_PromptState,
bundled_prompt,
fetch_prompt,
- model_experiment_prompt,
)
from products.signals.backend.temporal.report_safety_judge import REPORT_SAFETY_SYSTEM_ONE_PROMPT
from products.signals.backend.temporal.safety_filter import SIGNAL_SAFETY_SYSTEM_ONE_PROMPT
[email protected]("model", [JEVK_MODEL, DEFAULT_SYSTEM_ONE_MODEL])
-def test_managed_prompt_keeps_policy_model_question_and_threshold_together(model: str) -> None:
+def test_managed_prompt_keeps_policy_model_question_and_threshold_together() -> None:
fallback = bundled_prompt("signals-actionability-issue", "old {description}", "old question", 0.85)
policy = (
"When in doubt, classify as ACTIONABLE.\n"
@@ -34,7 +31,7 @@ def test_managed_prompt_keeps_policy_model_question_and_threshold_together(model
name=fallback.name,
version=2,
prompt=policy,
- config={"model": model, "question": "new question", "threshold": 0.91},
+ config={"model": DEFAULT_SYSTEM_ONE_MODEL, "question": "new question", "threshold": 0.91},
)
with (
@@ -47,7 +44,7 @@ def test_managed_prompt_keeps_policy_model_question_and_threshold_together(model
assert managed is not None
assert (managed.policy, managed.model, managed.question, managed.threshold, managed.version, managed.source) == (
policy,
- model,
+ DEFAULT_SYSTEM_ONE_MODEL,
"new question",
0.91,
2,
@@ -185,28 +182,6 @@ def test_failed_refresh_reverts_a_managed_safety_prompt_to_bundled() -> None:
cache._executor.shutdown(wait=True)
[email protected]("changed", [None, "policy", "question", "threshold", "model", "version", "source"])
-def test_model_experiment_requires_a_model_only_candidate(changed: str | None) -> None:
- primary = replace(SIGNAL_SAFETY_SYSTEM_ONE_PROMPT, model=JEVK_MODEL, source="managed", version=2)
- candidate = replace(primary, model="posthog/hogference/jeeves-0.1", version=3)
- if changed == "policy":
- candidate = replace(candidate, policy="different policy")
- elif changed == "question":
- candidate = replace(candidate, question="different question")
- elif changed == "threshold":
- candidate = replace(candidate, threshold=0.5)
- elif changed == "model":
- candidate = replace(candidate, model=primary.model)
- elif changed == "version":
- candidate = replace(candidate, version=4)
- elif changed == "source":
- candidate = replace(candidate, source="bundled")
- with patch("products.signals.backend.system_one_prompts.current_prompt", return_value=candidate):
- assert model_experiment_prompt(primary, 3, "posthog/hogference/jeeves-0.1") == (
- candidate if changed is None else None
- )
-
-
def test_versioned_shadow_refresh_does_not_replace_production() -> None:
fallback = SIGNAL_SAFETY_SYSTEM_ONE_PROMPT
primary = replace(fallback, source="managed", version=2)
@@ -222,3 +197,25 @@ def test_versioned_shadow_refresh_does_not_replace_production() -> None:
assert cache.current(fallback) == primary
finally:
cache._executor.shutdown(wait=True)
+
+
[email protected]("fallback", [SIGNAL_SAFETY_SYSTEM_ONE_PROMPT, REPORT_SAFETY_SYSTEM_ONE_PROMPT])
+def test_managed_jevk_prompt_is_rejected(fallback: SystemOnePrompt) -> None:
+ result = PromptResult(
+ source="api",
+ name=fallback.name,
+ version=1,
+ prompt=fallback.policy,
+ config={
+ "model": "posthog/hogference/jevk5-fp8-0.2",
+ "question": fallback.question,
+ "threshold": fallback.threshold,
+ },
+ )
+ with (
+ patch("products.signals.backend.system_one_prompts.posthoganalytics.personal_api_key", "test-key"),
+ patch("products.signals.backend.system_one_prompts.Prompts") as prompts,
+ ):
+ prompts.return_value.get.return_value = result
+ assert fetch_prompt(fallback) is None
+ assert fallback.model == DEFAULT_SYSTEM_ONE_MODEL