Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(ai-observability): run eval report signal summaries on the Go gateway only (#128207)

Jonathan Mieloo1b7450facd10
diff --git a/posthog/temporal/ai_observability/eval_reports/emit_signal.py b/posthog/temporal/ai_observability/eval_reports/emit_signal.py
index 8a5767dfce62..5c8795fe6d4e 100644
--- a/posthog/temporal/ai_observability/eval_reports/emit_signal.py
+++ b/posthog/temporal/ai_observability/eval_reports/emit_signal.py
@@ -30,6 +30,7 @@ import temporalio.workflow
from pydantic import BaseModel, Field
from temporalio.common import RetryPolicy
+from posthog.llm.gateway_client import GatewayNotConfiguredError, resolve_ai_gateway_config
from posthog.models import PropertyDefinition
from posthog.models.team import Team
from posthog.sync import database_sync_to_async
@@ -137,10 +138,13 @@ async def summarize_report_for_signal(
) -> EvalReportSignalSummary:
"""Use the signals LLM to produce a signal-sized summary of a report run.
- `ai_product` is the Go-gateway opt-in. A call without it routes to the legacy Python
- gateway under the bare `signals` slug. This tag stays separate from the report agent's
- `aio_eval_reports`: a budget denial on one must not stop the other.
+ The call runs only on the Go gateway, so it raises when the gateway is not configured rather
+ than reaching the legacy Python gateway under the bare `signals` slug. The `ai_product` tag
+ stays separate from the report agent's `aio_eval_reports`: a budget denial on one must not
+ stop the other.
"""
+ if resolve_ai_gateway_config() is None:
+ raise GatewayNotConfiguredError("AI_GATEWAY_URL and AI_GATEWAY_API_KEY must be configured")
user_prompt = _build_eval_report_signal_prompt(inputs, content)
def validate(text: str) -> EvalReportSignalSummary:
@@ -266,5 +270,7 @@ class EmitEvalReportSignalWorkflow(PostHogWorkflow):
emit_eval_report_signal_activity,
inputs,
schedule_to_close_timeout=timedelta(seconds=180),
- retry_policy=RetryPolicy(maximum_attempts=2),
+ retry_policy=RetryPolicy(
+ maximum_attempts=2, non_retryable_error_types=[GatewayNotConfiguredError.__name__]
+ ),
)
diff --git a/posthog/temporal/ai_observability/eval_reports/test/test_emit_signal.py b/posthog/temporal/ai_observability/eval_reports/test/test_emit_signal.py
index 83ec4ce0695f..c46a9f552352 100644
--- a/posthog/temporal/ai_observability/eval_reports/test/test_emit_signal.py
+++ b/posthog/temporal/ai_observability/eval_reports/test/test_emit_signal.py
@@ -3,9 +3,12 @@ import random
import pytest
from unittest.mock import AsyncMock, patch
+from django.test import override_settings
+
import pytest_asyncio
from asgiref.sync import sync_to_async
+from posthog.llm.gateway_client import GatewayNotConfiguredError
from posthog.models import Organization, PropertyDefinition, Team, User
from posthog.temporal.ai_observability.eval_reports.emit_signal import (
EmitEvalReportSignalInputs,
@@ -133,6 +136,7 @@ async def _setup_org_not_ai_approved(ateam):
# `ai_product` is the Go-gateway opt-in. Dropping it reverts this site to the Python gateway
# and unattributes its spend, with no call failing to show it.
@pytest.mark.asyncio
+@override_settings(AI_GATEWAY_URL="https://ai-gateway.test/v1", AI_GATEWAY_API_KEY="phs_test")
async def test_eval_report_summary_opts_in_as_aio_eval_reports_for_signals():
summary = EvalReportSignalSummary(title="Pass rate dropped", description="d", significance=0.5)
with patch(
@@ -147,6 +151,16 @@ async def test_eval_report_summary_opts_in_as_aio_eval_reports_for_signals():
assert kwargs["stage"] == "eval_report_signal_summary"
+@override_settings(AI_GATEWAY_URL="", AI_GATEWAY_API_KEY="")
+async def test_eval_report_summary_refuses_the_python_gateway_when_go_is_unconfigured():
+ with patch("posthog.temporal.ai_observability.eval_reports.emit_signal.call_llm", new=AsyncMock()) as summary_call:
+ with pytest.raises(GatewayNotConfiguredError):
+ await summarize_report_for_signal(_make_inputs(team_id=1), _make_content())
+
+ summary_call.assert_not_awaited()
+
+
@pytest.mark.asyncio
@pytest.mark.django_db
class TestEmitEvalReportSignalActivity: