:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(business-knowledge): meter chunk ingestion and knowledge queries (#127254)
Alex V3f5b31ea4de2
diff --git a/.semgrep/rules/security/idor-team-scoped-models.yaml b/.semgrep/rules/security/idor-team-scoped-models.yamlindex c29ddf22316a..84458111ef75 100644--- a/.semgrep/rules/security/idor-team-scoped-models.yaml+++ b/.semgrep/rules/security/idor-team-scoped-models.yaml@@ -270,6 +270,7 @@ rules:|KnowledgeLibraryRepository|KnowledgeLibrarySource|KnowledgeSource+ |KnowledgeUsageRecord|PlaygroundChat|PlaygroundTurn|Link@@ -774,6 +775,7 @@ rules:|KnowledgeLibraryRepository|KnowledgeLibrarySource|KnowledgeSource+ |KnowledgeUsageRecord|PlaygroundChat|PlaygroundTurn|Link@@ -1183,6 +1185,7 @@ rules:|KnowledgeChunk|KnowledgeDocument|KnowledgeSource+ |KnowledgeUsageRecord|Link|LiveDebuggerBreakpoint|LLMModelConfigurationdiff --git a/products/business_knowledge/backend/billing.py b/products/business_knowledge/backend/billing.pynew file mode 100644index 000000000000..d558f78d5106--- /dev/null+++ b/products/business_knowledge/backend/billing.py@@ -0,0 +1,80 @@+"""Meters business knowledge usage into the append-only `KnowledgeUsageRecord` ledger.++Two meters, matching the pricing RFC:++- `business_knowledge_chunks`: every chunk written, on any ingest path. A refresh that re-indexes changed+ content writes its chunks again, so it counts again. Content a refresh finds unchanged is not+ re-chunked, so it is not counted. Sources PostHog generates by learning from support tickets count+ the same way.+- `business_knowledge_queries`: every knowledge search made with a `RetrievalTrace`, on any customer+ surface. PostHog's internal docs-shadow comparison is not metered.++This is metering only. Nothing is sent to the usage report or the billing service yet, and no quota is+enforced, so a metering failure is logged and never fails the ingest or search that triggered it.+"""++from uuid import UUID++from django.db import transaction++import structlog++from .models import KnowledgeUsageKey, KnowledgeUsageRecord++logger = structlog.get_logger(__name__)++# PostHog's internal docs-search comparison; the customer never sees its results.+UNMETERED_SURFACES = frozenset({"docs_shadow"})+++def record_chunks_ingested(+ *,+ team_id: int,+ quantity: int,+ surface: str,+ source_id: UUID | None,+ document_id: UUID | None,+) -> None:+ """Record `quantity` chunks just written for `team_id`.++ Call inside the transaction that wrote the chunks, so a rollback removes the record with them.+ `team_id` must be the canonical team the chunks are stored under.+ """+ if quantity <= 0:+ return+ try:+ # Savepoint, so a failed insert can't poison the caller's transaction and roll back the chunks.+ with transaction.atomic():+ KnowledgeUsageRecord.objects.unscoped().create(+ team_id=team_id,+ usage_key=KnowledgeUsageKey.CHUNKS,+ quantity=quantity,+ surface=surface,+ source_id=source_id,+ document_id=document_id,+ )+ except Exception:+ logger.exception(+ "business_knowledge.usage_record_failed", usage_key=KnowledgeUsageKey.CHUNKS.value, team_id=team_id+ )+++def record_query(*, team_id: int, trace_id: str, surface: str) -> None:+ """Record one knowledge search for `team_id`.++ The trace id is the idempotency key: a retried Temporal activity that reuses its trace id writes no+ second row. `team_id` must be the canonical team. Searches from `UNMETERED_SURFACES` write nothing.+ """+ if surface in UNMETERED_SURFACES:+ return+ try:+ with transaction.atomic():+ KnowledgeUsageRecord.objects.unscoped().get_or_create(+ usage_key=KnowledgeUsageKey.QUERIES,+ idempotency_key=trace_id,+ defaults={"team_id": team_id, "quantity": 1, "surface": surface},+ )+ except Exception:+ logger.exception(+ "business_knowledge.usage_record_failed", usage_key=KnowledgeUsageKey.QUERIES.value, team_id=team_id+ )diff --git a/products/business_knowledge/backend/llm_telemetry.py b/products/business_knowledge/backend/llm_telemetry.pyindex a2ace172d719..97980f4248ab 100644--- a/products/business_knowledge/backend/llm_telemetry.py+++ b/products/business_knowledge/backend/llm_telemetry.py@@ -11,7 +11,7 @@ import mathimport dataclassesfrom collections.abc import Iterable, Mappingfrom typing import Literal-from uuid import UUID, uuid4+from uuid import UUID, uuid4, uuid5import structlogimport posthoganalytics@@ -36,7 +36,7 @@ EMBEDDING_COST_PROVIDER = "openai"# The Kafka embedding path returns no token usage, so ingestion estimates it.CHARS_PER_TOKEN_ESTIMATE = 4-RetrievalSurface = Literal["api", "support", "posthog_ai", "docs_shadow"]+RetrievalSurface = Literal["api", "support", "posthog_ai", "docs_shadow", "learning"]EventProperties = dict[str, str | int]@@ -45,9 +45,18 @@ class RetrievalTrace:"""One search. The query embedding and the rerank of that search share the trace."""surface: RetrievalSurface+ # Also the usage idempotency key: retryable work passes `stable_trace_id(...)`.trace_id: str = dataclasses.field(default_factory=lambda: str(uuid4()))+_TRACE_NAMESPACE = UUID("8c1f3b1e-6f0a-4d7e-9a43-2b5b0f6c1d27")+++def stable_trace_id(*parts: object) -> str:+ """A trace id that is the same every time it is built from the same parts."""+ return str(uuid5(_TRACE_NAMESPACE, ":".join(str(part) for part in parts)))++def ingest_properties(*, team_id: int, document_id: UUID, source_id: UUID, source_type: str, feature: str) -> EventProperties:diff --git a/products/business_knowledge/backend/logic.py b/products/business_knowledge/backend/logic.pyindex 70ab98781084..3e09e43335a7 100644--- a/products/business_knowledge/backend/logic.py+++ b/products/business_knowledge/backend/logic.py@@ -55,7 +55,7 @@ from posthog.ph_client import feature_enabled_or_falsefrom posthog.security.url_validation import is_url_allowedfrom posthog.sync import database_sync_to_async-from . import crawl, discover, file_parse, html_parse, llm_telemetry, url_fetch+from . import billing, crawl, discover, file_parse, html_parse, llm_telemetry, url_fetchfrom .constants import (BK_DRILLDOWN_DEFAULT_RADIUS,BK_DRILLDOWN_MAX_RADIUS,@@ -98,7 +98,10 @@ from .constants import (TRIAL_QUIET_PERIOD,)from .libraries import file_new_learned_source-from .llm_telemetry import RetrievalTrace+from .llm_telemetry import ( # re-exported for other products+ RetrievalTrace,+ stable_trace_id as stable_trace_id,+)from .models import (REFRESH_INTERVAL_TIMEDELTAS,AddedBy,@@ -357,6 +360,14 @@ def _bulk_create_chunks(KnowledgeChunk.objects.filter(team_id=team_id, id__in=[c.id for c in created]).update(content_search_vector=SearchVector("content", config="english"))+ # The one place chunks are written, so every ingest path is metered here.+ billing.record_chunks_ingested(+ team_id=team_id,+ quantity=len(created),+ surface="generated" if source.is_generated else source.source_type,+ source_id=source.id,+ document_id=document.id,+ )# --- Quota enforcement -------------------------------------------------------@@ -2718,7 +2729,6 @@ def _result_from_chunk(chunk: KnowledgeChunk) -> KnowledgeSearchResult:)-@with_team_scope(canonical=True)def search_knowledge(team_id: int,query: str,@@ -2727,6 +2737,43 @@ def search_knowledge(use_semantic: bool = False,query_embedding: list[float] | None = None,library_id: UUID | None = None,+ trace: RetrievalTrace | None = None,+ usage_team_id: int | None = None,+) -> list[KnowledgeSearchResult]:+ """+ Run one knowledge search, metered as one query when ``trace`` is given.++ Every product surface passes its ``RetrievalTrace``, so every retrieval is metered, whatever it+ returns, except PostHog's internal docs-shadow comparison (see ``billing.UNMETERED_SURFACES``).+ Only offline evals and tests search without a trace. A search that raises meters nothing.+ The query is metered to ``usage_team_id`` (the canonical team) when given, else to ``team_id``.+ """+ results = _search_knowledge(+ team_id,+ query,+ limit=limit,+ use_semantic=use_semantic,+ query_embedding=query_embedding,+ library_id=library_id,+ )+ if trace is not None:+ billing.record_query(+ team_id=usage_team_id if usage_team_id is not None else team_id,+ trace_id=trace.trace_id,+ surface=trace.surface,+ )+ return results+++@with_team_scope(canonical=True)+def _search_knowledge(+ team_id: int,+ query: str,+ *,+ limit: int = 10,+ use_semantic: bool = False,+ query_embedding: list[float] | None = None,+ library_id: UUID | None = None,) -> list[KnowledgeSearchResult]:"""Hybrid (lexical + semantic) relevance search over BK chunks.@@ -2884,6 +2931,7 @@ def search_knowledge_for_team(if library_id is not None:with team_scope(canonical_team_id, canonical=True):if not _library_has_sources(canonical_team_id, library_id):+ billing.record_query(team_id=canonical_team_id, trace_id=trace.trace_id, surface=trace.surface)return []embedding: list[float] | None = Nonetry:@@ -2899,6 +2947,7 @@ def search_knowledge_for_team(use_semantic=embedding is not None,query_embedding=embedding,library_id=library_id,+ trace=trace,)@@ -2910,6 +2959,8 @@ async def async_search_knowledge_for_team(limit: int = 10,) -> list[KnowledgeSearchResult]:"""Async hybrid search. Embedding failure or timeout falls back to full-text search."""+ # The search stays on `team.id` because callers only authorize `team`; usage belongs to the parent.+ usage_team_id = await database_sync_to_async(resolve_effective_team_id, thread_sensitive=False)(team.id)embedding: list[float] | None = Nonetry:response = await asyncio.wait_for(@@ -2926,6 +2977,8 @@ async def async_search_knowledge_for_team(limit=limit,use_semantic=embedding is not None,query_embedding=embedding,+ trace=trace,+ usage_team_id=usage_team_id,)diff --git a/products/business_knowledge/backend/migrations/0025_knowledge_usage_record.py b/products/business_knowledge/backend/migrations/0025_knowledge_usage_record.pynew file mode 100644index 000000000000..2b0e854bd3ee--- /dev/null+++ b/products/business_knowledge/backend/migrations/0025_knowledge_usage_record.py@@ -0,0 +1,52 @@+from django.db import migrations, models++import posthog.uuidt+++class Migration(migrations.Migration):+ dependencies = [+ ("business_knowledge", "0024_knowledge_libraries"),+ ]++ operations = [+ migrations.CreateModel(+ name="KnowledgeUsageRecord",+ fields=[+ (+ "id",+ models.UUIDField(default=posthog.uuidt.uuid7, editable=False, primary_key=True, serialize=False),+ ),+ ("team_id", models.BigIntegerField()),+ (+ "usage_key",+ models.CharField(+ choices=[+ ("business_knowledge_chunks", "Chunks ingested"),+ ("business_knowledge_queries", "Queries"),+ ],+ max_length=64,+ ),+ ),+ ("quantity", models.PositiveIntegerField()),+ ("surface", models.CharField(blank=True, default="", max_length=32)),+ ("idempotency_key", models.CharField(blank=True, max_length=128, null=True)),+ ("source_id", models.UUIDField(blank=True, null=True)),+ ("document_id", models.UUIDField(blank=True, null=True)),+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ],+ options={+ "db_table": "posthog_business_knowledge_knowledgeusagerecord",+ "indexes": [+ models.Index(fields=["created_at", "team_id"], name="bk_usage_created_team"),+ models.Index(fields=["team_id", "usage_key", "created_at"], name="bk_usage_team_key_created"),+ ],+ "constraints": [+ models.UniqueConstraint(+ condition=models.Q(("idempotency_key__isnull", False)),+ fields=("usage_key", "idempotency_key"),+ name="bk_usage_idempotency_key",+ )+ ],+ },+ ),+ ]diff --git a/products/business_knowledge/backend/migrations/max_migration.txt b/products/business_knowledge/backend/migrations/max_migration.txtindex 41cc182df651..8e9da83fb846 100644--- a/products/business_knowledge/backend/migrations/max_migration.txt+++ b/products/business_knowledge/backend/migrations/max_migration.txt@@ -1 +1 @@-0024_knowledge_libraries+0025_knowledge_usage_recorddiff --git a/products/business_knowledge/backend/models/__init__.py b/products/business_knowledge/backend/models/__init__.pyindex 69244e1ddba1..f2012a8b248a 100644--- a/products/business_knowledge/backend/models/__init__.py+++ b/products/business_knowledge/backend/models/__init__.py@@ -21,6 +21,7 @@ from .knowledge_library import KnowledgeLibraryfrom .knowledge_library_repository import KnowledgeLibraryRepositoryfrom .knowledge_library_source import KnowledgeLibrarySourcefrom .knowledge_source import KnowledgeSource+from .knowledge_usage_record import KnowledgeUsageKey, KnowledgeUsageRecordfrom .playground_chat import PlaygroundChatfrom .playground_turn import PlaygroundTurnfrom .team_business_knowledge_config import TeamBusinessKnowledgeConfig@@ -39,6 +40,8 @@ __all__ = ["KnowledgeLibraryRepository","KnowledgeLibrarySource","KnowledgeSource",+ "KnowledgeUsageKey",+ "KnowledgeUsageRecord","LearningProvider","PlaygroundChat","PlaygroundTurn",diff --git a/products/business_knowledge/backend/models/knowledge_usage_record.py b/products/business_knowledge/backend/models/knowledge_usage_record.pynew file mode 100644index 000000000000..e9e11fa20f8f--- /dev/null+++ b/products/business_knowledge/backend/models/knowledge_usage_record.py@@ -0,0 +1,54 @@+from django.db import models++from posthog.models.scoping.manager import TeamScopedManager+from posthog.models.utils import UUIDModel+++class KnowledgeUsageKey(models.TextChoices):+ """Billable meters. They must match the `usage_key` values in the billing service's plans config."""++ CHUNKS = "business_knowledge_chunks", "Chunks ingested"+ QUERIES = "business_knowledge_queries", "Queries"+++class KnowledgeUsageRecord(UUIDModel):+ """Append-only record of one unit of billable business knowledge work.++ Ingestion writes one row per chunk write, in the same transaction as the chunks, so a rolled-back+ write records nothing. Retrieval writes one row per search. Rows are never updated, and deleting the+ source, document or chunks they describe leaves them in place, so a delete never removes usage.++ Nothing reads this table for billing yet. It meters usage so the pricing can be checked against real+ data before the usage report and the billing service start counting it.+ """++ # Canonical team, no FK: deleting a team must not cascade into usage history.+ team_id = models.BigIntegerField()+ usage_key = models.CharField(max_length=64, choices=KnowledgeUsageKey.choices)+ quantity = models.PositiveIntegerField()+ surface = models.CharField(max_length=32, blank=True, default="")+ # The search's trace id, so a retried Temporal activity is counted once.+ idempotency_key = models.CharField(max_length=128, null=True, blank=True)+ # Plain ids, kept to answer "why was I billed for this?" after a delete.+ source_id = models.UUIDField(null=True, blank=True)+ document_id = models.UUIDField(null=True, blank=True)+ created_at = models.DateTimeField(auto_now_add=True)++ objects = TeamScopedManager()++ class Meta:+ db_table = "posthog_business_knowledge_knowledgeusagerecord"+ constraints = [+ models.UniqueConstraint(+ fields=["usage_key", "idempotency_key"],+ condition=models.Q(idempotency_key__isnull=False),+ name="bk_usage_idempotency_key",+ ),+ ]+ indexes = [+ models.Index(fields=["created_at", "team_id"], name="bk_usage_created_team"),+ models.Index(fields=["team_id", "usage_key", "created_at"], name="bk_usage_team_key_created"),+ ]++ def __str__(self) -> str:+ return f"{self.usage_key} x{self.quantity} (team {self.team_id})"diff --git a/products/business_knowledge/backend/temporal/learning/activities/analyze.py b/products/business_knowledge/backend/temporal/learning/activities/analyze.pyindex 7d0f425d84f5..1716993bb49e 100644--- a/products/business_knowledge/backend/temporal/learning/activities/analyze.py+++ b/products/business_knowledge/backend/temporal/learning/activities/analyze.py@@ -27,6 +27,7 @@ from products.business_knowledge.backend import learning_settings, logicfrom products.business_knowledge.backend.constants import BK_EMBEDDING_MODEL, BK_QUERY_EMBEDDING_TIMEOUTfrom products.business_knowledge.backend.learning.contracts import EvidenceBundle, EvidenceReffrom products.business_knowledge.backend.learning.providers import get_learning_provider+from products.business_knowledge.backend.llm_telemetry import RetrievalTrace, stable_trace_idfrom products.business_knowledge.backend.models import KnowledgeLearningRun, LearningRunResult, LearningRunStatusfrom ..constants import (@@ -475,7 +476,7 @@ def _contradiction_gate_rejection(return "none"-def _search_existing_knowledge(team: Team, query: str) -> list[logic.KnowledgeSearchResult]:+def _search_existing_knowledge(team: Team, query: str, *, run_id: UUID) -> list[logic.KnowledgeSearchResult]:embedding = generate_embedding(team,query,@@ -490,6 +491,8 @@ def _search_existing_knowledge(team: Team, query: str) -> list[logic.KnowledgeSelimit=LEARNING_SEARCH_LIMIT,use_semantic=True,query_embedding=embedding,+ # Keyed on the run, so a retried analysis is counted once.+ trace=RetrievalTrace(surface="learning", trace_id=stable_trace_id("learning", run_id)),)@@ -762,7 +765,7 @@ def _analyze(run: KnowledgeLearningRun, input: AnalyzeLearningEvidenceInput) ->return _finish_without_knowledge(run, rejection_code="pii")try:- results = _search_existing_knowledge(run.team, generated_text)+ results = _search_existing_knowledge(run.team, generated_text, run_id=run.id)except Exception:raise LearningAnalysisError("knowledge_search_failed") from Noneretrieved = _render_search_context(results)diff --git a/products/business_knowledge/backend/tests/test_billing.py b/products/business_knowledge/backend/tests/test_billing.pynew file mode 100644index 000000000000..c432de45bed6--- /dev/null+++ b/products/business_knowledge/backend/tests/test_billing.py@@ -0,0 +1,188 @@+import asyncio+from uuid import UUID++from posthog.test.base import BaseTest+from unittest.mock import patch++from django.db import transaction+from django.utils import timezone++from posthog.models.scoping import team_scope+from posthog.models.team import Team++from products.business_knowledge.backend import billing, logic+from products.business_knowledge.backend.llm_telemetry import RetrievalTrace, stable_trace_id+from products.business_knowledge.backend.models import (+ KnowledgeChunk,+ KnowledgeLibrary,+ KnowledgeUsageKey,+ KnowledgeUsageRecord,+)++TEXT = "Refunds are available within 30 days.\n\nShipping is free above 50 euros."+++class _UsageTest(BaseTest):+ def _rows(self, usage_key: KnowledgeUsageKey, team_id: int | None = None) -> list[KnowledgeUsageRecord]:+ return list(+ KnowledgeUsageRecord.objects.unscoped()+ .filter(team_id=team_id or self.team.id, usage_key=usage_key)+ .order_by("created_at")+ )+++class TestIngestionMetering(_UsageTest):+ def test_text_source_meters_every_chunk_it_writes(self) -> None:+ source = logic.create_text_source(team_id=self.team.id, created_by_id=self.user.id, name="Policies", text=TEXT)++ chunks = KnowledgeChunk.objects.unscoped().filter(source_id=source.id).count()+ assert chunks > 0+ assert [(r.quantity, r.surface, r.source_id) for r in self._rows(KnowledgeUsageKey.CHUNKS)] == [+ (chunks, "text", source.id)+ ]++ def test_editing_text_meters_the_new_chunks_again(self) -> None:+ source = logic.create_text_source(team_id=self.team.id, created_by_id=self.user.id, name="Policies", text=TEXT)++ logic.update_text_source(source_id=source.id, team_id=self.team.id, name=None, text=TEXT + "\n\nUpdated.")++ assert len(self._rows(KnowledgeUsageKey.CHUNKS)) == 2++ def test_learned_source_meters_as_generated_ingestion(self) -> None:+ result = logic.create_generated_knowledge_document(+ logic.CreateGeneratedKnowledgeDocument(+ team_id=self.team.id,+ provider="conversations",+ ticket_id=UUID("10000000-0000-0000-0000-000000000001"),+ ticket_number=42,+ source_team_id=self.team.id,+ resolution_comment_id=UUID("20000000-0000-0000-0000-000000000002"),+ analysis_version="post_resolution_v1",+ title="Refund policy",+ content="Refunds are available within 30 days.",+ evidence_revision_at=timezone.now(),+ )+ )++ rows = self._rows(KnowledgeUsageKey.CHUNKS)+ assert [(r.surface, r.source_id) for r in rows] == [("generated", result.source_id)]++ def test_a_rolled_back_write_meters_nothing(self) -> None:+ with self.assertRaises(RuntimeError), transaction.atomic():+ billing.record_chunks_ingested(+ team_id=self.team.id, quantity=3, surface="text", source_id=None, document_id=None+ )+ raise RuntimeError("rolled back")++ assert self._rows(KnowledgeUsageKey.CHUNKS) == []++ def test_zero_chunks_write_no_row(self) -> None:+ billing.record_chunks_ingested(+ team_id=self.team.id, quantity=0, surface="text", source_id=None, document_id=None+ )++ assert self._rows(KnowledgeUsageKey.CHUNKS) == []++ def test_a_metering_failure_does_not_fail_the_ingest(self) -> None:+ with patch.object(KnowledgeUsageRecord.objects, "unscoped", side_effect=RuntimeError("ledger down")):+ source = logic.create_text_source(+ team_id=self.team.id, created_by_id=self.user.id, name="Policies", text=TEXT+ )++ assert KnowledgeChunk.objects.unscoped().filter(source_id=source.id).exists()+ assert self._rows(KnowledgeUsageKey.CHUNKS) == []+++class TestRetrievalMetering(_UsageTest):+ def test_a_search_with_a_trace_meters_one_query(self) -> None:+ trace = RetrievalTrace(surface="api")++ logic.search_knowledge(self.team.id, "refunds", trace=trace)++ assert [(r.quantity, r.surface, r.idempotency_key) for r in self._rows(KnowledgeUsageKey.QUERIES)] == [+ (1, "api", trace.trace_id)+ ]++ def test_a_search_without_a_trace_is_not_metered(self) -> None:+ logic.search_knowledge(self.team.id, "refunds")++ assert self._rows(KnowledgeUsageKey.QUERIES) == []++ def test_the_internal_docs_shadow_search_is_not_metered(self) -> None:+ logic.search_knowledge(self.team.id, "refunds", trace=RetrievalTrace(surface="docs_shadow"))++ assert self._rows(KnowledgeUsageKey.QUERIES) == []++ def test_a_retried_search_with_the_same_trace_meters_once(self) -> None:+ trace = RetrievalTrace(surface="support", trace_id=stable_trace_id("workflow", "run", "activity", 0, "refunds"))++ logic.search_knowledge(self.team.id, "refunds", trace=trace)+ logic.search_knowledge(self.team.id, "refunds", trace=trace)++ assert len(self._rows(KnowledgeUsageKey.QUERIES)) == 1++ def test_a_search_against_an_empty_library_is_metered(self) -> None:+ with team_scope(self.team.id, canonical=True):+ library = KnowledgeLibrary.objects.create(team_id=self.team.id, name="Empty")+ trace = RetrievalTrace(surface="api")++ results = logic.search_knowledge_for_team(self.team, "refunds", trace=trace, library_id=library.id)++ assert results == []+ assert [r.idempotency_key for r in self._rows(KnowledgeUsageKey.QUERIES)] == [trace.trace_id]++ def test_a_search_that_raises_meters_nothing(self) -> None:+ with (+ patch.object(logic, "_search_knowledge", side_effect=RuntimeError("search failed")),+ self.assertRaises(RuntimeError),+ ):+ logic.search_knowledge(self.team.id, "refunds", trace=RetrievalTrace(surface="api"))++ assert self._rows(KnowledgeUsageKey.QUERIES) == []++ def test_a_metering_failure_does_not_fail_the_search(self) -> None:+ with patch.object(KnowledgeUsageRecord.objects, "unscoped", side_effect=RuntimeError("ledger down")):+ results = logic.search_knowledge(self.team.id, "refunds", trace=RetrievalTrace(surface="api"))++ assert results == []++ def test_sync_search_from_a_child_environment_meters_the_parent(self) -> None:+ child = Team.objects.create(+ organization=self.organization, parent_team=self.team, project=self.team.project, name="Child environment"+ )++ with patch.object(logic, "generate_embedding", side_effect=RuntimeError("no embeddings")):+ logic.search_knowledge_for_team(child, "refunds", trace=RetrievalTrace(surface="api"))++ assert len(self._rows(KnowledgeUsageKey.QUERIES)) == 1+ assert self._rows(KnowledgeUsageKey.QUERIES, team_id=child.id) == []++ def test_async_search_from_a_child_environment_searches_the_child_and_meters_the_parent(self) -> None:+ # PostHog AI only authorizes the environment it runs in, so the search must not widen to the parent.+ child = Team.objects.create(+ organization=self.organization, parent_team=self.team, project=self.team.project, name="Child environment"+ )++ def run_inline(fn, **_kwargs):+ async def call(*args, **kwargs):+ return fn(*args, **kwargs)++ return call++ with (+ patch.object(logic, "database_sync_to_async", side_effect=run_inline),+ patch.object(logic, "resolve_effective_team_id", return_value=self.team.id),+ patch.object(logic, "async_generate_embedding", side_effect=RuntimeError("no embeddings")),+ patch.object(logic, "_search_knowledge", return_value=[]) as search,+ patch.object(billing, "record_query") as record_query,+ ):+ asyncio.run(+ logic.async_search_knowledge_for_team(child, "refunds", trace=RetrievalTrace(surface="posthog_ai"))+ )++ assert search.call_args.args[0] == child.id+ assert record_query.call_args.kwargs["team_id"] == self.team.id++ def test_stable_trace_ids_repeat_for_the_same_parts_only(self) -> None:+ assert stable_trace_id("a", 1) == stable_trace_id("a", 1)+ assert stable_trace_id("a", 1) != stable_trace_id("a", 2)diff --git a/products/business_knowledge/backend/tests/test_learning_analyzer.py b/products/business_knowledge/backend/tests/test_learning_analyzer.pyindex 8aece4038ff3..c27c0a9171c2 100644--- a/products/business_knowledge/backend/tests/test_learning_analyzer.py+++ b/products/business_knowledge/backend/tests/test_learning_analyzer.py@@ -19,6 +19,7 @@ from posthog.models.team import Teamfrom products.business_knowledge.backend import learning_settings, logicfrom products.business_knowledge.backend.learning.contracts import EvidenceBundle, EvidenceRef, evidence_key_for+from products.business_knowledge.backend.llm_telemetry import RetrievalTrace, stable_trace_idfrom products.business_knowledge.backend.models import (KnowledgeDocument,KnowledgeLearningRun,@@ -455,6 +456,9 @@ class TestLearningAnalyzer:]assert search.call_args.kwargs["use_semantic"] is Trueassert search.call_args.kwargs["query_embedding"] == [0.1, 0.2]+ assert search.call_args.kwargs["trace"] == RetrievalTrace(+ surface="learning", trace_id=stable_trace_id("learning", run.id)+ )publish.assert_not_called()@pytest.mark.parametrize(diff --git a/products/conversations/backend/temporal/ai_reply/activities/retrieve.py b/products/conversations/backend/temporal/ai_reply/activities/retrieve.pyindex fdb059d26a3b..e63e31376580 100644--- a/products/conversations/backend/temporal/ai_reply/activities/retrieve.py+++ b/products/conversations/backend/temporal/ai_reply/activities/retrieve.py@@ -17,6 +17,7 @@ from products.business_knowledge.backend.logic import (get_document_window,rerank_chunks,search_knowledge_for_team,+ stable_trace_id,)from products.business_knowledge.backend.models import KnowledgeChunkfrom products.conversations.backend.temporal.ai_reply.constants import (@@ -34,17 +35,20 @@ logger = structlog.get_logger(__name__)@close_db_connectionsasync def support_retrieve_activity(input: RetrieveInput) -> RetrieveOutput:"""Search BK + rerank. On widen attempts, also fetch document windows around prior citations."""+ info = activity.info()+ # Stable across retries of this activity, so a retried search is counted once.+ trace_seed = f"{info.workflow_id}:{info.workflow_run_id}:{info.activity_id}"async with Heartbeater():- return await database_sync_to_async(_retrieve_sync, thread_sensitive=False)(input)+ return await database_sync_to_async(_retrieve_sync, thread_sensitive=False)(input, trace_seed)-def _retrieve_sync(input: RetrieveInput) -> RetrieveOutput:+def _retrieve_sync(input: RetrieveInput, trace_seed: str) -> RetrieveOutput:team = Team.objects.select_related("organization").get(id=input.team_id)all_results = []seen_chunk_ids: set[str] = set()- for query in input.queries:- trace = RetrievalTrace(surface="support")+ for index, query in enumerate(input.queries):+ trace = RetrievalTrace(surface="support", trace_id=stable_trace_id(trace_seed, index, query))results = search_knowledge_for_team(team, query, limit=RETRIEVE_LIMIT, trace=trace)reranked = rerank_chunks(team, query, results, top_k=RERANK_TOP_K, trace=trace)for r in reranked:diff --git a/products/conversations/backend/temporal/tests/test_retrieve_metering.py b/products/conversations/backend/temporal/tests/test_retrieve_metering.pynew file mode 100644index 000000000000..8f61f9376e01--- /dev/null+++ b/products/conversations/backend/temporal/tests/test_retrieve_metering.py@@ -0,0 +1,32 @@+from posthog.test.base import BaseTest+from unittest.mock import patch++from products.business_knowledge.backend import logic+from products.business_knowledge.backend.models import KnowledgeUsageKey, KnowledgeUsageRecord+from products.conversations.backend.temporal.ai_reply.activities.retrieve import _retrieve_sync+from products.conversations.backend.temporal.ai_reply.schemas import RetrieveInput++RETRIEVE_MODULE = "products.conversations.backend.temporal.ai_reply.activities.retrieve"+++class TestSupportRetrieveMetering(BaseTest):+ def _query_count(self) -> int:+ return (+ KnowledgeUsageRecord.objects.unscoped()+ .filter(team_id=self.team.id, usage_key=KnowledgeUsageKey.QUERIES)+ .count()+ )++ def test_each_query_is_metered_once_per_activity_even_when_retried(self) -> None:+ input = RetrieveInput(team_id=self.team.id, queries=["refunds", "shipping"])++ with (+ patch(f"{RETRIEVE_MODULE}.rerank_chunks", return_value=[]),+ patch.object(logic, "generate_embedding", side_effect=RuntimeError("no embeddings")),+ ):+ _retrieve_sync(input, "workflow:run-1:activity-1")+ _retrieve_sync(input, "workflow:run-1:activity-1")+ assert self._query_count() == 2++ _retrieve_sync(input, "workflow:run-2:activity-1")+ assert self._query_count() == 4