:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(subscriptions): route change summaries through the ai gateway (#127572)
Jonathan Mieloo1b3e373daab3
diff --git a/products/subscriptions/backend/temporal/llm_change_summary.py b/products/subscriptions/backend/temporal/llm_change_summary.pyindex 9459c583ef33..b0e490576f6b 100644--- a/products/subscriptions/backend/temporal/llm_change_summary.py+++ b/products/subscriptions/backend/temporal/llm_change_summary.py@@ -1,24 +1,21 @@from __future__ import annotations-import osimport reimport base64from dataclasses import dataclass-from typing import TYPE_CHECKING--from django.conf import settings+from typing import TYPE_CHECKING, castimport structlog-import posthoganalytics+from openai import OpenAIfrom openai.lib._pydantic import to_strict_json_schema-from posthoganalytics.ai.openai import OpenAI+from openai.types.chat import ChatCompletionMessageParamfrom prometheus_client import Counterif TYPE_CHECKING:from posthog.models.team.team import Team-from posthog.event_usage import groupsfrom posthog.exceptions_capture import capture_exception+from posthog.llm.gateway_client import GatewayNotConfiguredError, build_openai_client, resolve_ai_gateway_configfrom posthog.llm_prompt import normalize_prompt_to_stringfrom posthog.security.llm_prompt_sanitization import (INSIGHT_DESCRIPTION_MAX_LEN,@@ -391,10 +388,21 @@ def _attach_images_to_user_message(return AttachedImageSummary(len(ordered_ids), bytes_total, len(user_text))-def _get_openai_client(*, max_retries: int = 1) -> OpenAI:- if not os.environ.get("OPENAI_API_KEY"):- raise ValueError("OPENAI_API_KEY environment variable not set")- return OpenAI(posthog_client=posthoganalytics.setup(), base_url=settings.OPENAI_BASE_URL, max_retries=max_retries)+def _get_openai_client(*, team: Team | None, delivery_id: str | None, user_tag: str) -> OpenAI:+ if resolve_ai_gateway_config() is None:+ raise GatewayNotConfiguredError("AI_GATEWAY_URL and AI_GATEWAY_API_KEY must be configured")+ properties: dict[str, str] = {}+ if delivery_id:+ properties["delivery_id"] = delivery_id+ headers: dict[str, str] = {}+ if team is not None:+ properties["team_id"] = str(team.id)+ else:+ headers["X-PostHog-Billable"] = "false"+ client = build_openai_client(+ "subscriptions", ai_product="subscriptions", properties=properties, distinct_id=user_tag+ )+ return client.with_options(max_retries=1, default_headers=headers)def _previous_states_rendered_in_current_delivery(@@ -463,49 +471,37 @@ def generate_change_summary(user_message_length=attached.user_text_length,)- client = _get_openai_client()-instance_region = get_instance_region() or "HOBBY"user_tag = f"{instance_region}/subscription-summary-team-{team_id}"if delivery_id:user_tag = f"{user_tag}-delivery-{delivery_id}"- posthog_properties: dict[str, object] = {"ai_product": "subscriptions"}- if delivery_id:- posthog_properties["delivery_id"] = delivery_id-- extra_capture_kwargs: dict[str, object] = {}- if team is not None:- posthog_properties["$ai_billable"] = True- posthog_properties["team_id"] = team.id- extra_capture_kwargs["posthog_groups"] = {"project": str(team.id), **groups()}+ client = _get_openai_client(team=team, delivery_id=delivery_id, user_tag=user_tag)prompt_tokens, completion_tokens = 0, 0- for max_tokens in (1500, 4500):- result = client.chat.completions.create(- model="gpt-4.1-mini",- temperature=0.3,- max_tokens=max_tokens,- response_format={- "type": "json_schema",- "json_schema": {- "name": "subscription_summary",- "strict": True,- "schema": to_strict_json_schema(ReportSummary),+ with client:+ for max_tokens in (1500, 4500):+ result = client.chat.completions.create(+ model="gpt-4.1-mini",+ temperature=0.3,+ max_tokens=max_tokens,+ response_format={+ "type": "json_schema",+ "json_schema": {+ "name": "subscription_summary",+ "strict": True,+ "schema": to_strict_json_schema(ReportSummary),+ },},- },- timeout=20,- messages=messages,- user=user_tag,- posthog_distinct_id=user_tag,- posthog_properties=posthog_properties,- **extra_capture_kwargs,- )- if result.usage:- prompt_tokens += result.usage.prompt_tokens- completion_tokens += result.usage.completion_tokens- if not result.choices or result.choices[0].finish_reason != "length":- break+ timeout=20,+ messages=cast(list[ChatCompletionMessageParam], messages),+ user=user_tag,+ )+ if result.usage:+ prompt_tokens += result.usage.prompt_tokens+ completion_tokens += result.usage.completion_tokens+ if not result.choices or result.choices[0].finish_reason != "length":+ breakcontent: str = ""if result.choices and result.choices[0].message.content:diff --git a/products/subscriptions/backend/temporal/query_results_summary.py b/products/subscriptions/backend/temporal/query_results_summary.pyindex 89b060452042..f96cfd165461 100644--- a/products/subscriptions/backend/temporal/query_results_summary.py+++ b/products/subscriptions/backend/temporal/query_results_summary.py@@ -4,15 +4,12 @@ import jsonimport asynciofrom collections import Counterfrom datetime import datetime-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING, Any, cast-from django.conf import settings+from openai import APIConnectionError, APIStatusError, AsyncOpenAI+from openai.types.chat import ChatCompletionMessageParam-import posthoganalytics-from openai import APIConnectionError, APIStatusError-from posthoganalytics.ai.openai import AsyncOpenAI--from posthog.event_usage import groups+from posthog.llm.gateway_client import GatewayNotConfiguredError, build_async_openai_client, resolve_ai_gateway_configfrom posthog.security.llm_prompt_sanitization import sanitize_user_text, strip_llm_framing_markersfrom posthog.utils import get_instance_region@@ -85,8 +82,21 @@ def _event_statistics(query_kind: str, query_results: dict[str, Any]) -> dict[streturn statistics-def _get_async_openai_client() -> AsyncOpenAI:- return AsyncOpenAI(posthog_client=posthoganalytics.setup(), base_url=settings.OPENAI_BASE_URL, max_retries=0)+def _get_async_openai_client(*, team: Team | None, delivery_id: str | None, user_tag: str) -> AsyncOpenAI:+ if resolve_ai_gateway_config() is None:+ raise GatewayNotConfiguredError("AI_GATEWAY_URL and AI_GATEWAY_API_KEY must be configured")+ properties = {"summary_stage": "query_condensation"}+ if delivery_id:+ properties["delivery_id"] = delivery_id+ headers = {"X-PostHog-Privacy-Mode": "true"}+ if team is not None:+ properties["team_id"] = str(team.id)+ else:+ headers["X-PostHog-Billable"] = "false"+ client = build_async_openai_client(+ "subscriptions", ai_product="subscriptions", properties=properties, distinct_id=user_tag+ )+ return client.with_options(max_retries=0, default_headers=headers)async def condense_query_results(@@ -126,15 +136,8 @@ async def condense_query_results(]team_id = team.id if team else 0user_tag = f"{get_instance_region() or 'HOBBY'}/subscription-summary-team-{team_id}"- properties: dict[str, object] = {"ai_product": "subscriptions", "summary_stage": "query_condensation"}- if delivery_id:- properties["delivery_id"] = delivery_id- capture_kwargs: dict[str, object] = {}- if team:- properties.update({"$ai_billable": True, "team_id": team.id})- capture_kwargs["posthog_groups"] = {"project": str(team.id), **groups()}- async with _get_async_openai_client() as client:+ async with _get_async_openai_client(team=team, delivery_id=delivery_id, user_tag=user_tag) as client:for attempt in range(2):try:result = await client.chat.completions.create(@@ -142,12 +145,8 @@ async def condense_query_results(temperature=0,max_tokens=1_200,timeout=20,- messages=messages,+ messages=cast(list[ChatCompletionMessageParam], messages),user=user_tag,- posthog_distinct_id=user_tag,- posthog_properties=properties,- posthog_privacy_mode=True,- **capture_kwargs,)except (APIConnectionError, APIStatusError) as error:if (diff --git a/products/subscriptions/backend/temporal/test_llm_change_summary.py b/products/subscriptions/backend/temporal/test_llm_change_summary.pyindex 1051a30585b5..df804acc7351 100644--- a/products/subscriptions/backend/temporal/test_llm_change_summary.py+++ b/products/subscriptions/backend/temporal/test_llm_change_summary.py@@ -5,8 +5,12 @@ from types import SimpleNamespaceimport pytestfrom unittest.mock import MagicMock, patch+from django.test import override_settings+from parameterized import parameterized+from posthog.llm.gateway_client import GatewayNotConfiguredError+from products.subscriptions.backend.temporal.llm_change_summary import (_attach_images_to_user_message,_get_openai_client,@@ -529,28 +533,6 @@ class TestGenerateChangeSummary:assert "Do not use markdown formatting" not in user_promptassert "Start each bullet with •" not in user_prompt- @patch("posthog.event_usage.SITE_URL", "https://us.posthog.com")- @patch("products.subscriptions.backend.temporal.llm_change_summary._get_openai_client")- def test_generation_is_tagged_billable(self, mock_get_client):- mock_client = mock_get_client.return_value- mock_client.chat.completions.create.return_value = _mock_openai_response("- summary", 10, 5)-- team = MagicMock()- team.id = 42-- generate_change_summary(- [_make_state(1, "X", "data")],- [_make_state(1, "X", "data", timestamp="2025-04-15T10:00:00Z")],- team=team,- )-- call_kwargs = mock_client.chat.completions.create.call_args.kwargs- properties = call_kwargs["posthog_properties"]- assert properties["$ai_billable"] is True- assert properties["team_id"] == 42- assert properties["ai_product"] == "subscriptions"- assert call_kwargs["posthog_groups"] == {"project": "42", "instance": "https://us.posthog.com"}-@patch("products.subscriptions.backend.temporal.llm_change_summary._get_openai_client")def test_uses_initial_prompt_when_no_previous_states(self, mock_get_client):mock_client = mock_get_client.return_value@@ -698,9 +680,8 @@ class TestGenerateChangeSummary:user_tag = mock_client.chat.completions.create.call_args.kwargs["user"]assert user_tag.endswith("-delivery-abc-123")- @patch("posthog.event_usage.SITE_URL", "https://us.posthog.com")@patch("products.subscriptions.backend.temporal.llm_change_summary._get_openai_client")- def test_marks_generation_billable_to_team_when_team_provided(self, mock_get_client):+ def test_builds_the_client_for_the_delivery_team(self, mock_get_client):mock_client = mock_get_client.return_valuemock_client.chat.completions.create.return_value = _mock_openai_response("- ok")@@ -709,75 +690,47 @@ class TestGenerateChangeSummary:generate_change_summary(None, current, team=team, delivery_id="abc-123") # type: ignore[arg-type]- call_kwargs = mock_client.chat.completions.create.call_args.kwargs- assert call_kwargs["posthog_properties"]["$ai_billable"] is True- assert call_kwargs["posthog_properties"]["team_id"] == 42- assert call_kwargs["posthog_properties"]["ai_product"] == "subscriptions"- assert call_kwargs["posthog_properties"]["delivery_id"] == "abc-123"- assert call_kwargs["posthog_groups"] == {"project": "42", "instance": "https://us.posthog.com"}- assert call_kwargs["posthog_distinct_id"] == call_kwargs["user"]-- @patch("products.subscriptions.backend.temporal.llm_change_summary._get_openai_client")- def test_does_not_mark_billable_when_team_missing(self, mock_get_client):- mock_client = mock_get_client.return_value- mock_client.chat.completions.create.return_value = _mock_openai_response("- ok")-- current = [_make_state(1, "Pageviews", "...", timestamp="2025-04-15T10:00:00Z")]-- generate_change_summary(None, current, team=None)-- call_kwargs = mock_client.chat.completions.create.call_args.kwargs- assert "$ai_billable" not in call_kwargs["posthog_properties"]- assert "team_id" not in call_kwargs["posthog_properties"]- assert "posthog_groups" not in call_kwargs-- def test_get_openai_client_raises_when_api_key_missing(self, monkeypatch):- monkeypatch.delenv("OPENAI_API_KEY", raising=False)- with pytest.raises(ValueError, match="OPENAI_API_KEY"):- _get_openai_client()-- def test_emits_ai_generation_event_with_billing_properties(self, monkeypatch):- monkeypatch.setenv("OPENAI_API_KEY", "test-fake-key")- monkeypatch.setattr("posthog.event_usage.SITE_URL", "https://us.posthog.com")-- fake_client = MagicMock()- monkeypatch.setattr("posthoganalytics.default_client", fake_client)-- usage_details = MagicMock()- usage_details.cached_tokens = 0- usage_details.reasoning_tokens = 0- usage = MagicMock()- usage.prompt_tokens = 10- usage.completion_tokens = 5- usage.prompt_tokens_details = usage_details- usage.completion_tokens_details = usage_details- choice = MagicMock()- choice.message.content = json.dumps({"findings": [{"items": [{"kind": "text", "text": "ok"}]}]})- choice.message.tool_calls = None- choice.finish_reason = "stop"- fake_response = MagicMock()- fake_response.choices = [choice]- fake_response.usage = usage- fake_response.model = "gpt-4.1-mini"-- team = SimpleNamespace(id=42)- current = [_make_state(1, "Pageviews", "...", timestamp="2025-04-15T10:00:00Z")]+ user_tag = mock_client.chat.completions.create.call_args.kwargs["user"]+ assert mock_get_client.call_args.kwargs == {"team": team, "delivery_id": "abc-123", "user_tag": user_tag}- with patch("openai.resources.chat.completions.Completions.create", return_value=fake_response):- generate_change_summary(None, current, team=team, delivery_id="abc-123") # type: ignore[arg-type]+ @parameterized.expand(+ [+ ("team", SimpleNamespace(id=42), {"team_id": "42", "delivery_id": "abc-123"}, None),+ ("no_team", None, {"delivery_id": "abc-123"}, "false"),+ ]+ )+ @override_settings(AI_GATEWAY_URL="https://ai-gateway.example.com/v1", AI_GATEWAY_API_KEY="phs_test")+ def test_gateway_client_bills_the_team_or_opts_out(self, _name, team, expected_properties, expected_billable):+ client = _get_openai_client(team=team, delivery_id="abc-123", user_tag="US/subscription-summary-team-42")++ headers = client.default_headers+ assert str(client.base_url) == "https://ai-gateway.example.com/v1/"+ assert client.max_retries == 1+ assert headers["X-PostHog-Product"] == "subscriptions"+ assert json.loads(str(headers["X-PostHog-Properties"])) == {+ **expected_properties,+ "ai_product": "subscriptions",+ }+ assert headers["X-PostHog-Distinct-Id"] == "US/subscription-summary-team-42"+ assert headers.get("X-PostHog-Billable") == expected_billable+ assert "X-PostHog-Privacy-Mode" not in headers- captured_calls = [- c.kwargs for method in (fake_client.capture, fake_client.capture_ai) for c in method.call_args_list+ @parameterized.expand(+ [+ ("unset", "", ""),+ ("missing_key", "https://ai-gateway.example.com/v1", ""),+ ("missing_v1_path", "https://ai-gateway.example.com", "phs_test"),]- ai_generation_calls = [c for c in captured_calls if c.get("event") == "$ai_generation"]- assert len(ai_generation_calls) == 1, (- f"expected exactly one $ai_generation capture, got events: {[c.get('event') for c in captured_calls]}"- )- captured = ai_generation_calls[0]- assert captured["properties"]["$ai_billable"] is True- assert captured["properties"]["team_id"] == 42- assert captured["properties"]["ai_product"] == "subscriptions"- assert captured["groups"] == {"project": "42", "instance": "https://us.posthog.com"}+ )+ def test_refuses_the_python_gateway_when_go_is_unconfigured(self, _name, url, api_key):+ with override_settings(+ AI_GATEWAY_URL=url,+ AI_GATEWAY_API_KEY=api_key,+ LLM_GATEWAY_URL="https://llm-gateway.example.com",+ LLM_GATEWAY_API_KEY="phx_test",+ ):+ with pytest.raises(GatewayNotConfiguredError):+ _get_openai_client(team=None, delivery_id=None, user_tag="US/subscription-summary-team-42")@parameterized.expand([diff --git a/products/subscriptions/backend/temporal/test_query_results_summary.py b/products/subscriptions/backend/temporal/test_query_results_summary.pyindex 0af91b218f47..572ddbe36d1b 100644--- a/products/subscriptions/backend/temporal/test_query_results_summary.py+++ b/products/subscriptions/backend/temporal/test_query_results_summary.py@@ -4,12 +4,17 @@ from types import SimpleNamespaceimport pytestfrom unittest.mock import AsyncMock, MagicMock, patch+from django.test import override_settings+import httpxfrom openai import APIConnectionError, APIStatusErrorfrom parameterized import parameterized+from posthog.llm.gateway_client import GatewayNotConfiguredError+from products.subscriptions.backend.temporal.query_results_summary import (MAX_CONDENSATION_INPUT_LENGTH,+ _get_async_openai_client,condense_query_results,)@@ -82,7 +87,6 @@ class TestCondenseQueryResults:}assert payload["has_more"] is Trueassert payload["resolved_date_range"] == query_results["resolved_date_range"]- assert client.chat.completions.create.call_args.kwargs["posthog_privacy_mode"] is Trueasync def test_rejects_invalid_summary_without_cutting_it(self) -> None:client = MagicMock()@@ -143,3 +147,46 @@ class TestCondenseQueryResults:with pytest.raises(type(error)):await condense_query_results("EventsQuery", {"result": [["example_event"]]})assert client.chat.completions.create.call_count == (2 if retryable else 1)+++class TestCondensationGatewayClient:+ @parameterized.expand(+ [+ ("team", SimpleNamespace(id=42), {"team_id": "42"}, None),+ ("no_team", None, {}, "false"),+ ]+ )+ @override_settings(AI_GATEWAY_URL="https://ai-gateway.example.com/v1", AI_GATEWAY_API_KEY="phs_test")+ def test_bills_the_team_or_opts_out_and_keeps_privacy_mode(+ self, _name, team, expected_properties, expected_billable+ ) -> None:+ client = _get_async_openai_client(team=team, delivery_id=None, user_tag="US/subscription-summary-team-42")++ headers = client.default_headers+ assert str(client.base_url) == "https://ai-gateway.example.com/v1/"+ assert client.max_retries == 0+ assert headers["X-PostHog-Product"] == "subscriptions"+ assert json.loads(str(headers["X-PostHog-Properties"])) == {+ **expected_properties,+ "summary_stage": "query_condensation",+ "ai_product": "subscriptions",+ }+ assert headers["X-PostHog-Privacy-Mode"] == "true"+ assert headers.get("X-PostHog-Billable") == expected_billable++ @parameterized.expand(+ [+ ("unset", "", ""),+ ("missing_key", "https://ai-gateway.example.com/v1", ""),+ ("missing_v1_path", "https://ai-gateway.example.com", "phs_test"),+ ]+ )+ def test_refuses_the_python_gateway_when_go_is_unconfigured(self, _name, url, api_key) -> None:+ with override_settings(+ AI_GATEWAY_URL=url,+ AI_GATEWAY_API_KEY=api_key,+ LLM_GATEWAY_URL="https://llm-gateway.example.com",+ LLM_GATEWAY_API_KEY="phx_test",+ ):+ with pytest.raises(GatewayNotConfiguredError):+ _get_async_openai_client(team=None, delivery_id=None, user_tag="US/subscription-summary-team-42")