:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(aio): report usage for the AI events that PostHog generates (#121811)
Alex Maysc6ee716b434e
diff --git a/posthog/kafka_client/client.py b/posthog/kafka_client/client.pyindex dae81d0c73d4..8bb30f58567c 100644--- a/posthog/kafka_client/client.py+++ b/posthog/kafka_client/client.py@@ -365,6 +365,10 @@ class _KafkaProducer:return result+ def poll(self, timeout: float) -> int:+ """Run the delivery callbacks that are ready, waiting up to `timeout` seconds for one."""+ return self.producer.poll(timeout)+def flush(self, timeout: Optional[float] = None) -> int:# confluent's flush() returns the count still undelivered (0 once acked);# surface it so callers can detect a partial flush. It rejects a None@@ -488,3 +492,18 @@ class ClickhouseProducer:from posthog.kafka_client.routing import get_producerreturn get_producer(topic=topic).produce(topic=topic, data=data)++ def wait_for_delivery(self, results: list[ProduceResult], timeout: float) -> None:+ """Raise `TimeoutError` or `KafkaException` unless Kafka acknowledges every result within `timeout` seconds."""+ if not settings.TEST:+ from posthog.kafka_client.routing import get_producer++ # The producer runs delivery callbacks only while something polls it. Poll instead of flush,+ # because a flush also waits for every other message on the shared producer.+ producers = {result.topic: get_producer(topic=result.topic) for result in results}.values()+ deadline = time.monotonic() + timeout+ while not all(result.done() for result in results) and (remaining := deadline - time.monotonic()) > 0:+ for producer in producers:+ producer.poll(min(remaining, 0.1))+ for result in results:+ result.get(timeout=0)diff --git a/posthog/kafka_client/test/test_client.py b/posthog/kafka_client/test/test_client.pyindex 9677afde4647..37a13b2b35cb 100644--- a/posthog/kafka_client/test/test_client.py+++ b/posthog/kafka_client/test/test_client.py@@ -7,9 +7,10 @@ from unittest.mock import MagicMock, patchfrom django.test import SimpleTestCase, TestCase, override_settings+from confluent_kafka import KafkaError, KafkaExceptionfrom parameterized import parameterized-from posthog.kafka_client.client import _KafkaProducer+from posthog.kafka_client.client import ClickhouseProducer, ProduceResult, _KafkaProducerfrom posthog.settings.kafka import KafkaProfileSettings, _resolve_producer_settings@@ -259,6 +260,33 @@ class KafkaClientTestCase(TestCase):_ROUNDTRIP = os.environ.get("KAFKA_ROUNDTRIP_TESTS") == "1"+class ClickhouseProducerDeliveryTestCase(SimpleTestCase):+ @parameterized.expand(+ [+ ("acknowledged", True, None, None),+ ("failed", True, KafkaError(KafkaError._MSG_TIMED_OUT), KafkaException), # type: ignore[attr-defined]+ ("unacknowledged", False, None, TimeoutError),+ ]+ )+ @override_settings(TEST=False)+ def test_wait_for_delivery(+ self, _name: str, delivered: bool, error: KafkaError | None, expected: type[Exception] | None+ ) -> None:+ result = ProduceResult(topic="events_json")+ producer = MagicMock()+ if delivered:+ producer.poll.side_effect = lambda timeout: result.set_result(error, None)++ with patch("posthog.kafka_client.routing.get_producer", return_value=producer):+ if expected is None:+ ClickhouseProducer().wait_for_delivery([result], timeout=0.05)+ else:+ with pytest.raises(expected):+ ClickhouseProducer().wait_for_delivery([result], timeout=0.05)++ producer.flush.assert_not_called()++@pytest.mark.skipif(not _ROUNDTRIP, reason="KAFKA_ROUNDTRIP_TESTS=1 required to hit a real broker")@override_settings(TEST=False)class KafkaClientRoundtripTestCase(SimpleTestCase):diff --git a/posthog/models/event/util.py b/posthog/models/event/util.pyindex 4f88816bbe33..e3c677f4f610 100644--- a/posthog/models/event/util.py+++ b/posthog/models/event/util.py@@ -15,7 +15,7 @@ from drf_spectacular.utils import extend_schema_field, extend_schema_serializerfrom rest_framework import serializersfrom posthog.clickhouse.client import sync_execute-from posthog.kafka_client.client import ClickhouseProducer+from posthog.kafka_client.client import ClickhouseProducer, ProduceResultfrom posthog.kafka_client.topics import KAFKA_EVENTS_JSONfrom posthog.models.element.element import Element, chain_to_elements, elements_to_stringfrom posthog.models.event.sql import BULK_INSERT_EVENT_SQL, EVENTS_JSON_DATA_TABLE, INSERT_EVENT_SQL@@ -87,6 +87,7 @@ def create_event(group3_created_at: Optional[Union[datetime, str]] = None,group4_created_at: Optional[Union[datetime, str]] = None,person_mode: Literal["full", "propertyless", "force_upgrade"] = "full",+ delivery_timeout_seconds: float | None = None,) -> str:if properties is None:properties = {}@@ -126,15 +127,20 @@ def create_event("person_mode": person_mode,}p = ClickhouseProducer()+ results: list[ProduceResult] = []if TEST and settings.CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA:json_data = {**data,"properties": _json_dumps_for_clickhouse(properties),"person_properties": _json_dumps_for_clickhouse(person_properties),}- p.produce(topic=KAFKA_EVENTS_JSON, sql=INSERT_EVENT_SQL(table_name=EVENTS_JSON_DATA_TABLE), data=json_data)+ results.append(+ p.produce(topic=KAFKA_EVENTS_JSON, sql=INSERT_EVENT_SQL(table_name=EVENTS_JSON_DATA_TABLE), data=json_data)+ )if not (TEST and settings.CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA and _events_only_in_active_schema.get()):- p.produce(topic=KAFKA_EVENTS_JSON, sql=INSERT_EVENT_SQL(), data=data)+ results.append(p.produce(topic=KAFKA_EVENTS_JSON, sql=INSERT_EVENT_SQL(), data=data))+ if delivery_timeout_seconds is not None:+ p.wait_for_delivery(results, delivery_timeout_seconds)return str(event_uuid)diff --git a/posthog/temporal/ai_observability/eval_reports/activities.py b/posthog/temporal/ai_observability/eval_reports/activities.pyindex 38b132a4abed..a35127423f0b 100644--- a/posthog/temporal/ai_observability/eval_reports/activities.py+++ b/posthog/temporal/ai_observability/eval_reports/activities.py@@ -890,13 +890,13 @@ async def store_report_run_activity(def store() -> str:import uuid- from posthog.models.event.util import create_eventfrom posthog.models.team import Teamfrom posthog.temporal.ai_observability.eval_reports.report_agent.schema import ( # noqa: PLC0415 -- keeps report agent dependencies off the activity import pathEvalReportGenerationStatus,EvalReportMetrics,normalize_report_content_payload,)+ from posthog.temporal.ai_observability.generated_event_usage import create_generated_ai_eventfrom products.ai_observability.backend.models.evaluation_reports import EvaluationReportRun@@ -966,7 +966,7 @@ async def store_report_run_activity(})- create_event(+ create_generated_ai_event(event_uuid=uuid.uuid4(),event="$ai_evaluation_report",team=team,diff --git a/posthog/temporal/ai_observability/eval_reports/test/test_activities.py b/posthog/temporal/ai_observability/eval_reports/test/test_activities.pyindex 3ea65d1f18d9..fdc8fda8b0ef 100644--- a/posthog/temporal/ai_observability/eval_reports/test/test_activities.py+++ b/posthog/temporal/ai_observability/eval_reports/test/test_activities.py@@ -218,7 +218,7 @@ async def test_store_sentiment_report_emits_generic_metrics_only() -> None:return_value=report_run,),patch("posthog.models.team.Team.objects.get", return_value=MagicMock()),- patch("posthog.models.event.util.create_event") as create_event,+ patch("posthog.temporal.ai_observability.generated_event_usage.create_event") as create_event,):result = await store_report_run_activity(inputs)@@ -263,7 +263,7 @@ async def test_store_legacy_boolean_report_emits_normalized_generic_metrics() ->return_value=report_run,) as create_report_run,patch("posthog.models.team.Team.objects.get", return_value=MagicMock()),- patch("posthog.models.event.util.create_event") as create_event,+ patch("posthog.temporal.ai_observability.generated_event_usage.create_event") as create_event,):await store_report_run_activity(inputs)@@ -308,7 +308,7 @@ async def test_store_strips_nul_from_report_content() -> None:return_value=report_run,) as create_report_run,patch("posthog.models.team.Team.objects.get", return_value=MagicMock()),- patch("posthog.models.event.util.create_event") as create_event,+ patch("posthog.temporal.ai_observability.generated_event_usage.create_event") as create_event,):await store_report_run_activity(inputs)@@ -344,7 +344,7 @@ async def test_store_metrics_unavailable_report_omits_placeholder_metrics() -> Nreturn_value=report_run,) as create_report_run,patch("posthog.models.team.Team.objects.get", return_value=MagicMock()),- patch("posthog.models.event.util.create_event") as create_event,+ patch("posthog.temporal.ai_observability.generated_event_usage.create_event") as create_event,):await store_report_run_activity(inputs)diff --git a/posthog/temporal/ai_observability/evaluation_clustering/event_emission.py b/posthog/temporal/ai_observability/evaluation_clustering/event_emission.pyindex 2eb3b460c341..f2965948c532 100644--- a/posthog/temporal/ai_observability/evaluation_clustering/event_emission.py+++ b/posthog/temporal/ai_observability/evaluation_clustering/event_emission.py@@ -14,9 +14,9 @@ import dataclassesimport numpy as np-from posthog.models.event.util import create_eventfrom posthog.models.team import Teamfrom posthog.temporal.ai_observability.evaluation_clustering.constants import EVENT_NAME_EVALUATION_CLUSTERS+from posthog.temporal.ai_observability.generated_event_usage import create_generated_ai_eventfrom posthog.temporal.ai_observability.trace_clustering.event_emission import _build_cluster_datafrom posthog.temporal.ai_observability.trace_clustering.models import (ClusterAggregateMetrics,@@ -81,7 +81,7 @@ def emit_evaluation_cluster_events(if clustering_params:properties["$ai_clustering_params"] = dataclasses.asdict(clustering_params)- create_event(+ create_generated_ai_event(event_uuid=uuid.uuid4(),event=EVENT_NAME_EVALUATION_CLUSTERS,team=team,diff --git a/posthog/temporal/ai_observability/evaluation_clustering/tests/test_event_emission.py b/posthog/temporal/ai_observability/evaluation_clustering/tests/test_event_emission.pyindex 5994579a3a1c..449f84bfc0b3 100644--- a/posthog/temporal/ai_observability/evaluation_clustering/tests/test_event_emission.py+++ b/posthog/temporal/ai_observability/evaluation_clustering/tests/test_event_emission.py@@ -34,9 +34,7 @@ class TestEmitEvaluationClusterEvents:cluster_labels = {0: ClusterLabel(title="Factuality failures", description="- shared hallucination pattern")}item_timestamps = {"e1": "2026-04-15T12:00:00Z", "e2": "2026-04-15T12:05:00Z"}- with patch(- "posthog.temporal.ai_observability.evaluation_clustering.event_emission.create_event"- ) as mock_create_event:+ with patch("posthog.temporal.ai_observability.generated_event_usage.create_event") as mock_create_event:clusters = emit_evaluation_cluster_events(team_id=mock_team.id,clustering_run_id="run-1",@@ -93,9 +91,7 @@ class TestEmitEvaluationClusterEvents:)}- with patch(- "posthog.temporal.ai_observability.evaluation_clustering.event_emission.create_event"- ) as mock_create_event:+ with patch("posthog.temporal.ai_observability.generated_event_usage.create_event") as mock_create_event:clusters = emit_evaluation_cluster_events(team_id=mock_team.id,clustering_run_id="run-1",diff --git a/posthog/temporal/ai_observability/generated_event_usage.py b/posthog/temporal/ai_observability/generated_event_usage.pynew file mode 100644index 000000000000..e3cc3d253555--- /dev/null+++ b/posthog/temporal/ai_observability/generated_event_usage.py@@ -0,0 +1,73 @@+import json+import uuid+import hashlib+from typing import Any++from django.utils import timezone++import structlog+from confluent_kafka import KafkaException++from posthog.models.event.util import create_event+from posthog.models.team import Team+from posthog.usage_ingestion.client import UsageRecord, report_usage++PRODUCER_ID = "ai-observability"+AI_EVENTS_USAGE_KEY = "ai_events"+EVENT_DELIVERY_TIMEOUT_SECONDS = 2.0++logger = structlog.get_logger(__name__)+++def create_generated_ai_event(+ *,+ event_uuid: uuid.UUID,+ event: str,+ team: Team,+ distinct_id: str,+ properties: dict[str, Any],+) -> None:+ """Write an AI event that PostHog generates in the team's project, and report its usage after Kafka acknowledges it.++ These events go straight to ClickHouse, so they skip the ingestion pipeline that writes the+ usage record for every other AI event. The nightly report counts them as AI events, so they+ need their own record here.+ """+ timestamp = timezone.now()+ try:+ create_event(+ event_uuid=event_uuid,+ event=event,+ team=team,+ distinct_id=distinct_id,+ timestamp=timestamp,+ properties=properties,+ delivery_timeout_seconds=EVENT_DELIVERY_TIMEOUT_SECONDS,+ )+ except (KafkaException, TimeoutError):+ # Usage counts only the events that Kafka acknowledged, as ingestion does. The error does not+ # fail the activity, because a retry repeats the work that produced the event.+ logger.warning("generated_ai_event_not_delivered", ai_event=event, team_id=team.id, exc_info=True)+ return+ report_usage(+ [+ UsageRecord(+ record_id=_record_id(timestamp.date().isoformat(), event, distinct_id, event_uuid),+ producer_id=PRODUCER_ID,+ team_id=team.id,+ usage_key=AI_EVENTS_USAGE_KEY,+ unit="events",+ quantity=1,+ timestamp_ms=int(timestamp.timestamp() * 1000),+ )+ ],+ site="ai_observability_generated_event",+ )+++def _record_id(day: str, event: str, distinct_id: str, event_uuid: uuid.UUID) -> str:+ # The same identity as the ingestion pipeline's records (`analyticsRecordId` in+ # nodejs/src/ingestion/common/steps/usage-records-steps.ts): the event's day, then a hash of+ # the event name, distinct ID and UUID.+ identity = json.dumps([event, distinct_id, str(event_uuid)], separators=(",", ":"), ensure_ascii=False)+ return f"{day}:{hashlib.sha256(identity.encode()).hexdigest()[:32]}"diff --git a/posthog/temporal/ai_observability/tests/test_generated_event_usage.py b/posthog/temporal/ai_observability/tests/test_generated_event_usage.pynew file mode 100644index 000000000000..85c76aafc043--- /dev/null+++ b/posthog/temporal/ai_observability/tests/test_generated_event_usage.py@@ -0,0 +1,59 @@+import re+import uuid+from contextlib import nullcontext++import pytest+from unittest.mock import MagicMock, patch++from confluent_kafka import KafkaError, KafkaException++from posthog.temporal.ai_observability.generated_event_usage import create_generated_ai_event+++@patch("posthog.temporal.ai_observability.generated_event_usage.report_usage")+@patch("posthog.temporal.ai_observability.generated_event_usage.create_event")+def test_reports_one_ai_event_record_for_each_generated_event(create_event: MagicMock, report_usage: MagicMock) -> None:+ team = MagicMock(id=42)+ first_uuid, second_uuid = uuid.uuid4(), uuid.uuid4()+ for event_uuid in (first_uuid, second_uuid):+ create_generated_ai_event(+ event_uuid=event_uuid,+ event="$ai_trace_summary",+ team=team,+ distinct_id="trace_summary_42",+ properties={"$ai_trace_id": "trace-1"},+ )++ records = [call.args[0][0] for call in report_usage.call_args_list]+ assert [(r.team_id, r.usage_key, r.unit, r.quantity) for r in records] == [(42, "ai_events", "events", 1)] * 2+ timestamp = create_event.call_args_list[0].kwargs["timestamp"]+ assert create_event.call_args_list[0].kwargs["delivery_timeout_seconds"] is not None+ assert re.fullmatch(rf"{timestamp.date().isoformat()}:[0-9a-f]{{32}}", records[0].record_id)+ assert records[0].timestamp_ms == int(timestamp.timestamp() * 1000)+ assert records[0].record_id != records[1].record_id+++ "error, raises",+ [+ (RuntimeError("write failed"), True),+ (KafkaException(KafkaError(KafkaError._MSG_TIMED_OUT)), False), # type: ignore[attr-defined]+ (TimeoutError("no acknowledgement"), False),+ ],+)+@patch("posthog.temporal.ai_observability.generated_event_usage.report_usage")+@patch("posthog.temporal.ai_observability.generated_event_usage.create_event")+def test_reports_no_usage_when_the_event_write_fails(+ create_event: MagicMock, report_usage: MagicMock, error: Exception, raises: bool+) -> None:+ create_event.side_effect = error+ with pytest.raises(type(error)) if raises else nullcontext():+ create_generated_ai_event(+ event_uuid=uuid.uuid4(),+ event="$ai_trace_summary",+ team=MagicMock(id=42),+ distinct_id="trace_summary_42",+ properties={},+ )++ report_usage.assert_not_called()diff --git a/posthog/temporal/ai_observability/trace_clustering/event_emission.py b/posthog/temporal/ai_observability/trace_clustering/event_emission.pyindex 6810b28e57ad..dc0227bdd2d0 100644--- a/posthog/temporal/ai_observability/trace_clustering/event_emission.py+++ b/posthog/temporal/ai_observability/trace_clustering/event_emission.py@@ -14,8 +14,8 @@ from django.utils.dateparse import parse_datetimeimport numpy as np-from posthog.models.event.util import create_eventfrom posthog.models.team import Team+from posthog.temporal.ai_observability.generated_event_usage import create_generated_ai_eventfrom posthog.temporal.ai_observability.trace_clustering import constantsfrom posthog.temporal.ai_observability.trace_clustering.constants import NOISE_CLUSTER_IDfrom posthog.temporal.ai_observability.trace_clustering.data import fetch_item_summaries@@ -147,7 +147,7 @@ def emit_cluster_events(if clustering_params:properties["$ai_clustering_params"] = dataclasses.asdict(clustering_params)- create_event(+ create_generated_ai_event(event_uuid=event_uuid,event=event_name,team=team,diff --git a/posthog/temporal/ai_observability/trace_clustering/tests/test_event_emission.py b/posthog/temporal/ai_observability/trace_clustering/tests/test_event_emission.pyindex 006208db4479..dacf61c8327c 100644--- a/posthog/temporal/ai_observability/trace_clustering/tests/test_event_emission.py+++ b/posthog/temporal/ai_observability/trace_clustering/tests/test_event_emission.py@@ -262,7 +262,7 @@ class TestBuildClusterData:class TestEmitClusterEvents:- @patch("posthog.temporal.ai_observability.trace_clustering.event_emission.create_event")+ @patch("posthog.temporal.ai_observability.generated_event_usage.create_event")@patch("posthog.temporal.ai_observability.trace_clustering.event_emission.fetch_item_summaries")def test_emits_event_with_correct_properties(self, mock_fetch_summaries, mock_create_event, mock_team):mock_fetch_summaries.return_value = {@@ -300,7 +300,7 @@ class TestEmitClusterEvents:assert call_kwargs["properties"]["$ai_clustering_job_id"] == "123"assert call_kwargs["properties"]["$ai_clustering_job_name"] == "OpenAI traces"- @patch("posthog.temporal.ai_observability.trace_clustering.event_emission.create_event")+ @patch("posthog.temporal.ai_observability.generated_event_usage.create_event")@patch("posthog.temporal.ai_observability.trace_clustering.event_emission.fetch_item_summaries")def test_returns_cluster_data_list(self, mock_fetch_summaries, mock_create_event, mock_team):mock_fetch_summaries.return_value = {}diff --git a/posthog/temporal/ai_observability/trace_summarization/summarize_and_save.py b/posthog/temporal/ai_observability/trace_summarization/summarize_and_save.pyindex e1a2133daff1..028636cdc301 100644--- a/posthog/temporal/ai_observability/trace_summarization/summarize_and_save.py+++ b/posthog/temporal/ai_observability/trace_summarization/summarize_and_save.py@@ -7,9 +7,9 @@ from uuid import uuid4import structlogimport temporalio-from posthog.models.event.util import create_eventfrom posthog.models.team import Teamfrom posthog.redis import get_async_client+from posthog.temporal.ai_observability.generated_event_usage import create_generated_ai_eventfrom posthog.temporal.ai_observability.trace_summarization import constantsfrom posthog.temporal.ai_observability.trace_summarization.models import (SummarizationActivityResult,@@ -69,7 +69,7 @@ def _save_trace_summary_event("trace_timestamp": ctx.trace_first_timestamp,}- create_event(+ create_generated_ai_event(event_uuid=uuid4(),event=constants.EVENT_NAME_TRACE_SUMMARY,team=ctx.team,@@ -101,7 +101,7 @@ def _save_generation_summary_event("trace_timestamp": ctx.trace_first_timestamp,}- create_event(+ create_generated_ai_event(event_uuid=uuid4(),event=constants.EVENT_NAME_GENERATION_SUMMARY,team=ctx.team,diff --git a/posthog/temporal/ai_observability/trace_summarization/tests/test_summarize_and_save.py b/posthog/temporal/ai_observability/trace_summarization/tests/test_summarize_and_save.pyindex fd6c43aaa13d..af9a7c80de92 100644--- a/posthog/temporal/ai_observability/trace_summarization/tests/test_summarize_and_save.py+++ b/posthog/temporal/ai_observability/trace_summarization/tests/test_summarize_and_save.py@@ -101,9 +101,7 @@ class TestSummarizeAndSaveActivity:patch("posthog.temporal.ai_observability.trace_summarization.summarize_and_save.summarize") as mock_summarize,- patch(- "posthog.temporal.ai_observability.trace_summarization.summarize_and_save.create_event"- ) as mock_create_event,+ patch("posthog.temporal.ai_observability.generated_event_usage.create_event") as mock_create_event,patch("posthog.temporal.ai_observability.trace_summarization.summarize_and_save.LLMTracesSummarizerEmbedder") as mock_embedder_class,@@ -155,9 +153,7 @@ class TestSummarizeAndSaveActivity:patch("posthog.temporal.ai_observability.trace_summarization.summarize_and_save.summarize") as mock_summarize,- patch(- "posthog.temporal.ai_observability.trace_summarization.summarize_and_save.create_event"- ) as mock_create_event,+ patch("posthog.temporal.ai_observability.generated_event_usage.create_event") as mock_create_event,patch("posthog.temporal.ai_observability.trace_summarization.summarize_and_save.LLMTracesSummarizerEmbedder") as mock_embedder_class,@@ -207,7 +203,7 @@ class TestSummarizeAndSaveActivity:"posthog.temporal.ai_observability.trace_summarization.summarize_and_save.summarize",side_effect=record_thread,),- patch("posthog.temporal.ai_observability.trace_summarization.summarize_and_save.create_event"),+ patch("posthog.temporal.ai_observability.generated_event_usage.create_event"),patch("posthog.temporal.ai_observability.trace_summarization.summarize_and_save.LLMTracesSummarizerEmbedder"),@@ -232,7 +228,7 @@ class TestSummarizeAndSaveActivity:patch("posthog.temporal.ai_observability.trace_summarization.summarize_and_save.summarize") as mock_summarize,- patch("posthog.temporal.ai_observability.trace_summarization.summarize_and_save.create_event"),+ patch("posthog.temporal.ai_observability.generated_event_usage.create_event"),patch("posthog.temporal.ai_observability.trace_summarization.summarize_and_save.LLMTracesSummarizerEmbedder") as mock_embedder_class,