:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(surveys): route survey LLM calls through the Go ai-gateway (#124912)
Jonathan Mieloob1c67625bd18
diff --git a/ee/surveys/summaries/headline_summary.py b/ee/surveys/summaries/headline_summary.pyindex 1898d598746e..72b1ef7d07ac 100644--- a/ee/surveys/summaries/headline_summary.py+++ b/ee/surveys/summaries/headline_summary.py@@ -2,7 +2,7 @@ from collections import Counterfrom datetime import datetimeimport structlog-from langchain_core.messages import HumanMessage, SystemMessage+from openai import OpenAIfrom posthog.schema import SurveyQuestionType@@ -11,14 +11,13 @@ from posthog.hogql.parser import parse_selectfrom posthog.api.utils import ServerTimingsGatheredfrom posthog.hogql_queries.paginators import HogQLHasMorePaginator+from posthog.llm.gateway_client import build_openai_client, resolve_ai_gateway_configfrom posthog.models import Team, Userfrom products.surveys.backend.models import Surveyfrom products.surveys.backend.responses.fetch_rows import RESPONSE_EVENT_FILTER, SUBMISSION_GROUPING_KEYfrom products.surveys.backend.util import get_archived_response_uuids, get_survey_response_clickhouse_query-from ee.hogai.llm import MaxChatOpenAI-logger = structlog.get_logger(__name__)RESPONSE_LIMIT = 100@@ -177,36 +176,24 @@ def generate_survey_headline(with timer("llm"):logger.info("[survey_headline_summary] starting LLM call", survey_id=str(survey.id))try:- llm = MaxChatOpenAI(- user=user,- team=team,+ completion = _headline_client(survey, team, user).chat.completions.create(model="gpt-4.1-mini",temperature=0.3,- billable=False,- inject_context=True,- streaming=False,- disable_streaming=True,+ messages=[+ {"role": "system", "content": HEADLINE_SYSTEM_PROMPT},+ {+ "role": "user",+ "content": HEADLINE_USER_PROMPT.format(+ survey_name=survey.name,+ formatted_data=formatted_data,+ ),+ },+ ],+ user=user.distinct_id or "",)-- messages = [- SystemMessage(content=HEADLINE_SYSTEM_PROMPT),- HumanMessage(- content=HEADLINE_USER_PROMPT.format(- survey_name=survey.name,- formatted_data=formatted_data,- )- ),- ]-- llm_result = llm.invoke(messages)logger.info("[survey_headline_summary] LLM call completed", survey_id=str(survey.id))- content = llm_result.content- if isinstance(content, str):- headline = content.strip()- elif content:- headline = str(content[0]).strip() if content else "Unable to generate summary"- else:- headline = "Unable to generate summary"+ content = completion.choices[0].message.content if completion.choices else None+ headline = content.strip() if content else "Unable to generate summary"except Exception as e:logger.exception("[survey_headline_summary] LLM call failed", survey_id=str(survey.id), error=str(e))raise@@ -219,3 +206,15 @@ def generate_survey_headline("has_more": has_more,"timings_header": timer.to_header_string(),}+++def _headline_client(survey: Survey, team: Team, user: User) -> OpenAI:+ if resolve_ai_gateway_config() is None:+ return OpenAI()+ client = build_openai_client(+ product="django",+ ai_product="surveys",+ distinct_id=user.distinct_id,+ properties={"ai_feature": "survey_headline", "survey_id": str(survey.id), "team_id": str(team.id)},+ )+ return client.with_options(default_headers={"X-PostHog-Billable": "false", "X-PostHog-Privacy-Mode": "true"})diff --git a/ee/surveys/summaries/summarize_surveys.py b/ee/surveys/summaries/summarize_surveys.pydeleted file mode 100644index 0422102d3aaa..000000000000--- a/ee/surveys/summaries/summarize_surveys.py+++ /dev/null@@ -1,154 +0,0 @@-from datetime import datetime-from typing import Optional, cast--import structlog-import posthoganalytics-from posthoganalytics.ai.openai import OpenAI-from prometheus_client import Histogram--from posthog.hogql import ast-from posthog.hogql.parser import parse_expr, parse_select--from posthog.api.utils import ServerTimingsGathered-from posthog.hogql_queries.paginators import HogQLHasMorePaginator-from posthog.models import Team, User-from posthog.utils import get_instance_region--from products.surveys.backend.responses.fetch_rows import RESPONSE_EVENT_FILTER, SUBMISSION_GROUPING_KEY--logger = structlog.get_logger(__name__)--TOKENS_IN_PROMPT_HISTOGRAM = Histogram(- "posthog_survey_summary_tokens_in_prompt_histogram",- "histogram of the number of tokens in the prompt used to generate a survey summary",- buckets=[- 0,- 10,- 50,- 100,- 500,- 1000,- 2000,- 3000,- 4000,- 5000,- 6000,- 7000,- 8000,- 10000,- 20000,- 30000,- 40000,- 50000,- 100000,- 128000,- float("inf"),- ],-)---def summarize_survey_responses(- survey_id: str,- question_text: str,- question_index: Optional[int],- question_id: Optional[str],- survey_start: datetime,- survey_end: datetime,- team: Team,- user: User,-):- timer = ServerTimingsGathered()-- with timer("prepare_query"):- paginator = HogQLHasMorePaginator(limit=100, offset=0)- q = parse_select(- """- SELECT argMaxIf(answer, timestamp, isNotNull(answer)) AS response- FROM (- SELECT getSurveyResponse({question_index}, {question_id}) AS answer,- timestamp, {submission_key} AS submission_key- FROM events- WHERE {response_events}- AND properties.$survey_id = {survey_id}- AND timestamp >= {start_date}- AND timestamp <= {end_date}- ) GROUP BY submission_key- HAVING length(trim(coalesce(response, ''))) > 0- """,- {- "survey_id": ast.Constant(value=survey_id),- "response_events": parse_expr(RESPONSE_EVENT_FILTER),- "submission_key": parse_expr(SUBMISSION_GROUPING_KEY),- "survey_response_property": ast.Constant(- value=f"$survey_response_{question_index}" if question_index else "$survey_response"- ),- "start_date": ast.Constant(value=survey_start),- "end_date": ast.Constant(value=survey_end),- "question_index": ast.Constant(value=question_index),- "question_id": ast.Constant(value=question_id),- },- )-- with timer("run_query"):- query_response = paginator.execute_hogql_query(- team=team,- query_type="survey_response_list_query",- query=cast(ast.SelectQuery, q),- )-- with timer("llm_api_prep"):- instance_region = get_instance_region() or "HOBBY"- prepared_data = [x[0] for x in query_response.results if x[0]]-- with timer("openai_completion"):- result = OpenAI(posthog_client=posthoganalytics.default_client).chat.completions.create(- model="gpt-4.1-mini", # allows 128k tokens- temperature=0.7,- messages=[- {- "role": "system",- "content": """- You are a product manager's assistant. You summarise survey responses from users for the product manager.- You don't do any other tasks.- """,- },- {- "role": "user",- "content": f"""the survey question is {question_text}.""",- },- {- "role": "user",- "content": f"""the survey responses are {prepared_data}.""",- },- {- "role": "user",- "content": """- generate a one or two paragraph summary of the survey response,-taking into consideration the survey question being asked.- only summarize, the goal is to identify real user pain points and needs.- use bullet points to identify the themes, and highlight quotes to bring them to life.- we're trying to identify what to work on.- use as concise and simple language as possible.- generate no text other than the summary.- the aim is to let people see themes in the responses received.- return the text in markdown format without using any paragraph formatting""",- },- ],- user=f"{instance_region}/{user.pk}",- posthog_privacy_mode=True,- posthog_distinct_id=user.distinct_id,- posthog_properties={- "ai_product": "surveys",- "ai_feature": "response-summary",- "team_id": team.id,- },- )-- usage = result.usage.prompt_tokens if result.usage else None- if usage:- TOKENS_IN_PROMPT_HISTOGRAM.observe(usage)-- logger.info("survey_summary_response", result=result)-- content: str = result.choices[0].message.content or ""- return {"content": content, "timings_header": timer.to_header_string()}diff --git a/products/surveys/backend/api/survey.py b/products/surveys/backend/api/survey.pyindex 856858dee52b..1428f66e0bd9 100644--- a/products/surveys/backend/api/survey.py+++ b/products/surveys/backend/api/survey.py@@ -85,6 +85,7 @@ from products.feature_flags.backend.facade.config import ConfigFormatErrorfrom products.feature_flags.backend.models.feature_flag import FeatureFlagfrom products.feature_flags.backend.ownership import FLAG_OWNER_SURVEY, assert_flag_available_forfrom products.product_analytics.backend.facade.models import Insight+from products.surveys.backend.llm import gemini_configuredfrom products.surveys.backend.models import (MAX_ITERATION_COUNT,MAX_ITERATION_FREQUENCY_DAYS,@@ -3206,8 +3207,7 @@ class SurveyViewSet(ApprovalHandlingMixin, TeamAndOrgViewSetMixin, AccessControlreturn Response(cached_response)environment_is_allowed = settings.DEBUG or is_cloud()- has_gemini_api_key = bool(settings.GEMINI_API_KEY)- if not environment_is_allowed or not has_gemini_api_key:+ if not environment_is_allowed or not gemini_configured():raise exceptions.ValidationError("survey summary is only supported in PostHog Cloud")# Same AI-processing gate that summary_headline uses — applies to per-question summaries too.@@ -3417,8 +3417,10 @@ class SurveyViewSet(ApprovalHandlingMixin, TeamAndOrgViewSetMixin, AccessControl"survey translation generation is only supported in PostHog Cloud or DEBUG mode")- if not settings.GEMINI_API_KEY:- raise exceptions.ValidationError("GEMINI_API_KEY must be configured to generate translations")+ if not gemini_configured():+ raise exceptions.ValidationError(+ "GEMINI_API_KEY or the AI gateway must be configured to generate translations"+ )if not self.team.organization.is_ai_data_processing_approved:return Response(diff --git a/products/surveys/backend/llm/__init__.py b/products/surveys/backend/llm/__init__.pyindex 63bd5788f64c..787906bd8849 100644--- a/products/surveys/backend/llm/__init__.py+++ b/products/surveys/backend/llm/__init__.py@@ -1,5 +1,5 @@"""Shared LLM utilities for surveys."""-from .client import create_gemini_client, generate_structured_output+from .client import create_gemini_client, gemini_configured, generate_structured_output-__all__ = ["create_gemini_client", "generate_structured_output"]+__all__ = ["create_gemini_client", "gemini_configured", "generate_structured_output"]diff --git a/products/surveys/backend/llm/client.py b/products/surveys/backend/llm/client.pyindex 82f18935b0b2..a1cae11bd8f7 100644--- a/products/surveys/backend/llm/client.py+++ b/products/surveys/backend/llm/client.py@@ -7,17 +7,24 @@ from django.conf import settingsimport structlogimport posthoganalytics-from google.genai.types import GenerateContentConfig+from google.genai.types import GenerateContentConfig, HttpOptionsfrom posthoganalytics.ai.gemini import genaifrom pydantic import BaseModelfrom rest_framework import exceptionsfrom posthog.event_usage import groups+from posthog.llm.gateway_client import build_gemini_client, resolve_ai_gateway_configlogger = structlog.get_logger(__name__)T = TypeVar("T", bound=BaseModel)+AI_PRODUCT = "surveys"+++def gemini_configured() -> bool:+ return bool(settings.GEMINI_API_KEY) or resolve_ai_gateway_config() is not None+def create_gemini_client():if settings.DEBUG and posthoganalytics.disabled:@@ -46,8 +53,6 @@ def generate_structured_output(distinct_id: str | None = None,billable: bool = False,) -> tuple[T, str]:- client = create_gemini_client()-config = GenerateContentConfig(system_instruction=system_prompt,response_mime_type="application/json",@@ -63,15 +68,26 @@ def generate_structured_output(posthog_groups.update(groups())try:- response = client.models.generate_content(- model=model,- contents=user_prompt,- config=config,- posthog_distinct_id=distinct_id or "",- posthog_trace_id=trace_id,- posthog_properties=properties,- posthog_groups=posthog_groups,+ gateway_client = build_gemini_client(+ ai_product=AI_PRODUCT,+ trace_id=trace_id,+ properties=_gateway_labels(properties, team_id),+ distinct_id=distinct_id,)+ if gateway_client:+ if not billable:+ config.http_options = HttpOptions(headers={"X-PostHog-Billable": "false"})+ response = gateway_client.models.generate_content(model=model, contents=user_prompt, config=config)+ else:+ response = create_gemini_client().models.generate_content(+ model=model,+ contents=user_prompt,+ config=config,+ posthog_distinct_id=distinct_id or "",+ posthog_trace_id=trace_id,+ posthog_properties=properties,+ posthog_groups=posthog_groups,+ )if not response.text:raise exceptions.ValidationError("Gemini returned empty response")@@ -83,3 +99,14 @@ def generate_structured_output(except Exception:logger.exception("Gemini API call failed", model=model, properties=properties)raise exceptions.APIException("Failed to generate response")+++def _gateway_labels(properties: dict, team_id: int | None) -> dict[str, str]:+ labels = {+ key: str(value)+ for key, value in properties.items()+ if value is not None and key != "ai_product" and not key.startswith("$")+ }+ if team_id is not None:+ labels["team_id"] = str(team_id)+ return labelsdiff --git a/products/surveys/backend/summarization/test_headline_summary.py b/products/surveys/backend/summarization/test_headline_summary.pynew file mode 100644index 000000000000..059447723774--- /dev/null+++ b/products/surveys/backend/summarization/test_headline_summary.py@@ -0,0 +1,29 @@+from unittest.mock import MagicMock, patch++from django.test import override_settings++from openai import OpenAI++from ee.surveys.summaries.headline_summary import _headline_client+++@patch("ee.surveys.summaries.headline_summary.build_openai_client")+def test_headline_calls_openai_directly_without_the_ai_gateway(mock_build_openai_client: MagicMock) -> None:+ with (+ override_settings(AI_GATEWAY_URL="", AI_GATEWAY_API_KEY=""),+ patch.dict("os.environ", {"OPENAI_API_KEY": "sk-test"}),+ ):+ client = _headline_client(MagicMock(id="survey-1"), MagicMock(id=7), MagicMock(distinct_id="user-1"))++ assert isinstance(client, OpenAI)+ mock_build_openai_client.assert_not_called()+++def test_headline_through_the_ai_gateway_is_private_and_unbilled() -> None:+ with override_settings(AI_GATEWAY_URL="https://gateway.test/v1", AI_GATEWAY_API_KEY="phs_test"):+ client = _headline_client(MagicMock(id="survey-1"), MagicMock(id=7), MagicMock(distinct_id="user-1"))++ assert str(client.base_url).startswith("https://gateway.test/v1")+ assert client.default_headers["X-PostHog-Billable"] == "false"+ assert client.default_headers["X-PostHog-Privacy-Mode"] == "true"+ assert client.default_headers["X-PostHog-Product"] == "surveys"diff --git a/products/surveys/backend/summarization/test_summarization.py b/products/surveys/backend/summarization/test_summarization.pyindex 849fbc01a1fc..25452a174bd8 100644--- a/products/surveys/backend/summarization/test_summarization.py+++ b/products/surveys/backend/summarization/test_summarization.py@@ -7,6 +7,8 @@ from unittest.mock import MagicMock, patchfrom rest_framework import exceptions+from products.surveys.backend.llm import generate_structured_output+from .formatting import format_as_markdownfrom .llm.gemini import SummarizationResult, summarize_with_geminifrom .llm.schema import SurveySummaryResponse, SurveyTheme@@ -162,3 +164,57 @@ class TestSummarizeWithGemini:properties = mock_client.models.generate_content.call_args.kwargs["posthog_properties"]assert "$ai_billable" not in properties++ @patch("products.surveys.backend.llm.client.create_gemini_client")+ @patch("products.surveys.backend.llm.client.build_gemini_client")+ def test_gateway_client_used_when_configured(self, mock_build_gateway, mock_create_client):+ gateway_client = MagicMock()+ mock_build_gateway.return_value = gateway_client+ valid_response = {+ "overview": "Users want better performance",+ "themes": [{"theme": "Speed", "description": "Fast loading times", "frequency": ">50%"}],+ "key_insight": "Focus on performance",+ }+ gateway_client.models.generate_content.return_value = MagicMock(text=json.dumps(valid_response))++ result = summarize_with_gemini("What do you want?", ["Make it faster"], team_id=42, distinct_id="user-1")++ mock_create_client.assert_not_called()+ assert not any(key.startswith("posthog_") for key in gateway_client.models.generate_content.call_args.kwargs)+ assert mock_build_gateway.call_args.kwargs == {+ "ai_product": "surveys",+ "trace_id": result.trace_id,+ "properties": {+ "ai_feature": "survey_summary",+ "response_count": "1",+ "team_id": "42",+ },+ "distinct_id": "user-1",+ }++ @pytest.mark.parametrize("billable,expected_headers", [(True, None), (False, {"X-PostHog-Billable": "false"})])+ @patch("products.surveys.backend.llm.client.build_gemini_client")+ def test_gateway_unbilled_call_opts_out_of_billing(self, mock_build_gateway, billable, expected_headers):+ gateway_client = MagicMock()+ mock_build_gateway.return_value = gateway_client+ gateway_client.models.generate_content.return_value = MagicMock(+ text=json.dumps(+ {+ "overview": "o",+ "themes": [{"theme": "t", "description": "d", "frequency": ">50%"}],+ "key_insight": "k",+ }+ )+ )++ generate_structured_output(+ model="gemini-3-flash-preview",+ system_prompt="system",+ user_prompt="user",+ response_schema=SurveySummaryResponse,+ team_id=42,+ billable=billable,+ )++ http_options = gateway_client.models.generate_content.call_args.kwargs["config"].http_options+ assert (http_options.headers if http_options else None) == expected_headers