:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(ai-observability): run eval report signal summaries on the Go gateway only (#128207)
Jonathan Mieloo1b7450facd10
diff --git a/posthog/temporal/ai_observability/eval_reports/emit_signal.py b/posthog/temporal/ai_observability/eval_reports/emit_signal.pyindex 8a5767dfce62..5c8795fe6d4e 100644--- a/posthog/temporal/ai_observability/eval_reports/emit_signal.py+++ b/posthog/temporal/ai_observability/eval_reports/emit_signal.py@@ -30,6 +30,7 @@ import temporalio.workflowfrom pydantic import BaseModel, Fieldfrom temporalio.common import RetryPolicy+from posthog.llm.gateway_client import GatewayNotConfiguredError, resolve_ai_gateway_configfrom posthog.models import PropertyDefinitionfrom posthog.models.team import Teamfrom posthog.sync import database_sync_to_async@@ -137,10 +138,13 @@ async def summarize_report_for_signal() -> EvalReportSignalSummary:"""Use the signals LLM to produce a signal-sized summary of a report run.- `ai_product` is the Go-gateway opt-in. A call without it routes to the legacy Python- gateway under the bare `signals` slug. This tag stays separate from the report agent's- `aio_eval_reports`: a budget denial on one must not stop the other.+ The call runs only on the Go gateway, so it raises when the gateway is not configured rather+ than reaching the legacy Python gateway under the bare `signals` slug. The `ai_product` tag+ stays separate from the report agent's `aio_eval_reports`: a budget denial on one must not+ stop the other."""+ if resolve_ai_gateway_config() is None:+ raise GatewayNotConfiguredError("AI_GATEWAY_URL and AI_GATEWAY_API_KEY must be configured")user_prompt = _build_eval_report_signal_prompt(inputs, content)def validate(text: str) -> EvalReportSignalSummary:@@ -266,5 +270,7 @@ class EmitEvalReportSignalWorkflow(PostHogWorkflow):emit_eval_report_signal_activity,inputs,schedule_to_close_timeout=timedelta(seconds=180),- retry_policy=RetryPolicy(maximum_attempts=2),+ retry_policy=RetryPolicy(+ maximum_attempts=2, non_retryable_error_types=[GatewayNotConfiguredError.__name__]+ ),)diff --git a/posthog/temporal/ai_observability/eval_reports/test/test_emit_signal.py b/posthog/temporal/ai_observability/eval_reports/test/test_emit_signal.pyindex 83ec4ce0695f..c46a9f552352 100644--- a/posthog/temporal/ai_observability/eval_reports/test/test_emit_signal.py+++ b/posthog/temporal/ai_observability/eval_reports/test/test_emit_signal.py@@ -3,9 +3,12 @@ import randomimport pytestfrom unittest.mock import AsyncMock, patch+from django.test import override_settings+import pytest_asynciofrom asgiref.sync import sync_to_async+from posthog.llm.gateway_client import GatewayNotConfiguredErrorfrom posthog.models import Organization, PropertyDefinition, Team, Userfrom posthog.temporal.ai_observability.eval_reports.emit_signal import (EmitEvalReportSignalInputs,@@ -133,6 +136,7 @@ async def _setup_org_not_ai_approved(ateam):# `ai_product` is the Go-gateway opt-in. Dropping it reverts this site to the Python gateway# and unattributes its spend, with no call failing to show it.@pytest.mark.asyncio+@override_settings(AI_GATEWAY_URL="https://ai-gateway.test/v1", AI_GATEWAY_API_KEY="phs_test")async def test_eval_report_summary_opts_in_as_aio_eval_reports_for_signals():summary = EvalReportSignalSummary(title="Pass rate dropped", description="d", significance=0.5)with patch(@@ -147,6 +151,16 @@ async def test_eval_report_summary_opts_in_as_aio_eval_reports_for_signals():assert kwargs["stage"] == "eval_report_signal_summary"+@override_settings(AI_GATEWAY_URL="", AI_GATEWAY_API_KEY="")+async def test_eval_report_summary_refuses_the_python_gateway_when_go_is_unconfigured():+ with patch("posthog.temporal.ai_observability.eval_reports.emit_signal.call_llm", new=AsyncMock()) as summary_call:+ with pytest.raises(GatewayNotConfiguredError):+ await summarize_report_for_signal(_make_inputs(team_id=1), _make_content())++ summary_call.assert_not_awaited()++@pytest.mark.asyncio@pytest.mark.django_dbclass TestEmitEvalReportSignalActivity: