Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(business-knowledge): meter chunk ingestion and knowledge queries (#127254)

Alex V3f5b31ea4de2
diff --git a/.semgrep/rules/security/idor-team-scoped-models.yaml b/.semgrep/rules/security/idor-team-scoped-models.yaml
index c29ddf22316a..84458111ef75 100644
--- a/.semgrep/rules/security/idor-team-scoped-models.yaml
+++ b/.semgrep/rules/security/idor-team-scoped-models.yaml
@@ -270,6 +270,7 @@ rules:
|KnowledgeLibraryRepository
|KnowledgeLibrarySource
|KnowledgeSource
+ |KnowledgeUsageRecord
|PlaygroundChat
|PlaygroundTurn
|Link
@@ -774,6 +775,7 @@ rules:
|KnowledgeLibraryRepository
|KnowledgeLibrarySource
|KnowledgeSource
+ |KnowledgeUsageRecord
|PlaygroundChat
|PlaygroundTurn
|Link
@@ -1183,6 +1185,7 @@ rules:
|KnowledgeChunk
|KnowledgeDocument
|KnowledgeSource
+ |KnowledgeUsageRecord
|Link
|LiveDebuggerBreakpoint
|LLMModelConfiguration
diff --git a/products/business_knowledge/backend/billing.py b/products/business_knowledge/backend/billing.py
new file mode 100644
index 000000000000..d558f78d5106
--- /dev/null
+++ b/products/business_knowledge/backend/billing.py
@@ -0,0 +1,80 @@
+"""Meters business knowledge usage into the append-only `KnowledgeUsageRecord` ledger.
+
+Two meters, matching the pricing RFC:
+
+- `business_knowledge_chunks`: every chunk written, on any ingest path. A refresh that re-indexes changed
+ content writes its chunks again, so it counts again. Content a refresh finds unchanged is not
+ re-chunked, so it is not counted. Sources PostHog generates by learning from support tickets count
+ the same way.
+- `business_knowledge_queries`: every knowledge search made with a `RetrievalTrace`, on any customer
+ surface. PostHog's internal docs-shadow comparison is not metered.
+
+This is metering only. Nothing is sent to the usage report or the billing service yet, and no quota is
+enforced, so a metering failure is logged and never fails the ingest or search that triggered it.
+"""
+
+from uuid import UUID
+
+from django.db import transaction
+
+import structlog
+
+from .models import KnowledgeUsageKey, KnowledgeUsageRecord
+
+logger = structlog.get_logger(__name__)
+
+# PostHog's internal docs-search comparison; the customer never sees its results.
+UNMETERED_SURFACES = frozenset({"docs_shadow"})
+
+
+def record_chunks_ingested(
+ *,
+ team_id: int,
+ quantity: int,
+ surface: str,
+ source_id: UUID | None,
+ document_id: UUID | None,
+) -> None:
+ """Record `quantity` chunks just written for `team_id`.
+
+ Call inside the transaction that wrote the chunks, so a rollback removes the record with them.
+ `team_id` must be the canonical team the chunks are stored under.
+ """
+ if quantity <= 0:
+ return
+ try:
+ # Savepoint, so a failed insert can't poison the caller's transaction and roll back the chunks.
+ with transaction.atomic():
+ KnowledgeUsageRecord.objects.unscoped().create(
+ team_id=team_id,
+ usage_key=KnowledgeUsageKey.CHUNKS,
+ quantity=quantity,
+ surface=surface,
+ source_id=source_id,
+ document_id=document_id,
+ )
+ except Exception:
+ logger.exception(
+ "business_knowledge.usage_record_failed", usage_key=KnowledgeUsageKey.CHUNKS.value, team_id=team_id
+ )
+
+
+def record_query(*, team_id: int, trace_id: str, surface: str) -> None:
+ """Record one knowledge search for `team_id`.
+
+ The trace id is the idempotency key: a retried Temporal activity that reuses its trace id writes no
+ second row. `team_id` must be the canonical team. Searches from `UNMETERED_SURFACES` write nothing.
+ """
+ if surface in UNMETERED_SURFACES:
+ return
+ try:
+ with transaction.atomic():
+ KnowledgeUsageRecord.objects.unscoped().get_or_create(
+ usage_key=KnowledgeUsageKey.QUERIES,
+ idempotency_key=trace_id,
+ defaults={"team_id": team_id, "quantity": 1, "surface": surface},
+ )
+ except Exception:
+ logger.exception(
+ "business_knowledge.usage_record_failed", usage_key=KnowledgeUsageKey.QUERIES.value, team_id=team_id
+ )
diff --git a/products/business_knowledge/backend/llm_telemetry.py b/products/business_knowledge/backend/llm_telemetry.py
index a2ace172d719..97980f4248ab 100644
--- a/products/business_knowledge/backend/llm_telemetry.py
+++ b/products/business_knowledge/backend/llm_telemetry.py
@@ -11,7 +11,7 @@ import math
import dataclasses
from collections.abc import Iterable, Mapping
from typing import Literal
-from uuid import UUID, uuid4
+from uuid import UUID, uuid4, uuid5
import structlog
import posthoganalytics
@@ -36,7 +36,7 @@ EMBEDDING_COST_PROVIDER = "openai"
# The Kafka embedding path returns no token usage, so ingestion estimates it.
CHARS_PER_TOKEN_ESTIMATE = 4
-RetrievalSurface = Literal["api", "support", "posthog_ai", "docs_shadow"]
+RetrievalSurface = Literal["api", "support", "posthog_ai", "docs_shadow", "learning"]
EventProperties = dict[str, str | int]
@@ -45,9 +45,18 @@ class RetrievalTrace:
"""One search. The query embedding and the rerank of that search share the trace."""
surface: RetrievalSurface
+ # Also the usage idempotency key: retryable work passes `stable_trace_id(...)`.
trace_id: str = dataclasses.field(default_factory=lambda: str(uuid4()))
+_TRACE_NAMESPACE = UUID("8c1f3b1e-6f0a-4d7e-9a43-2b5b0f6c1d27")
+
+
+def stable_trace_id(*parts: object) -> str:
+ """A trace id that is the same every time it is built from the same parts."""
+ return str(uuid5(_TRACE_NAMESPACE, ":".join(str(part) for part in parts)))
+
+
def ingest_properties(
*, team_id: int, document_id: UUID, source_id: UUID, source_type: str, feature: str
) -> EventProperties:
diff --git a/products/business_knowledge/backend/logic.py b/products/business_knowledge/backend/logic.py
index 70ab98781084..3e09e43335a7 100644
--- a/products/business_knowledge/backend/logic.py
+++ b/products/business_knowledge/backend/logic.py
@@ -55,7 +55,7 @@ from posthog.ph_client import feature_enabled_or_false
from posthog.security.url_validation import is_url_allowed
from posthog.sync import database_sync_to_async
-from . import crawl, discover, file_parse, html_parse, llm_telemetry, url_fetch
+from . import billing, crawl, discover, file_parse, html_parse, llm_telemetry, url_fetch
from .constants import (
BK_DRILLDOWN_DEFAULT_RADIUS,
BK_DRILLDOWN_MAX_RADIUS,
@@ -98,7 +98,10 @@ from .constants import (
TRIAL_QUIET_PERIOD,
)
from .libraries import file_new_learned_source
-from .llm_telemetry import RetrievalTrace
+from .llm_telemetry import ( # re-exported for other products
+ RetrievalTrace,
+ stable_trace_id as stable_trace_id,
+)
from .models import (
REFRESH_INTERVAL_TIMEDELTAS,
AddedBy,
@@ -357,6 +360,14 @@ def _bulk_create_chunks(
KnowledgeChunk.objects.filter(team_id=team_id, id__in=[c.id for c in created]).update(
content_search_vector=SearchVector("content", config="english")
)
+ # The one place chunks are written, so every ingest path is metered here.
+ billing.record_chunks_ingested(
+ team_id=team_id,
+ quantity=len(created),
+ surface="generated" if source.is_generated else source.source_type,
+ source_id=source.id,
+ document_id=document.id,
+ )
# --- Quota enforcement -------------------------------------------------------
@@ -2718,7 +2729,6 @@ def _result_from_chunk(chunk: KnowledgeChunk) -> KnowledgeSearchResult:
)
-@with_team_scope(canonical=True)
def search_knowledge(
team_id: int,
query: str,
@@ -2727,6 +2737,43 @@ def search_knowledge(
use_semantic: bool = False,
query_embedding: list[float] | None = None,
library_id: UUID | None = None,
+ trace: RetrievalTrace | None = None,
+ usage_team_id: int | None = None,
+) -> list[KnowledgeSearchResult]:
+ """
+ Run one knowledge search, metered as one query when ``trace`` is given.
+
+ Every product surface passes its ``RetrievalTrace``, so every retrieval is metered, whatever it
+ returns, except PostHog's internal docs-shadow comparison (see ``billing.UNMETERED_SURFACES``).
+ Only offline evals and tests search without a trace. A search that raises meters nothing.
+ The query is metered to ``usage_team_id`` (the canonical team) when given, else to ``team_id``.
+ """
+ results = _search_knowledge(
+ team_id,
+ query,
+ limit=limit,
+ use_semantic=use_semantic,
+ query_embedding=query_embedding,
+ library_id=library_id,
+ )
+ if trace is not None:
+ billing.record_query(
+ team_id=usage_team_id if usage_team_id is not None else team_id,
+ trace_id=trace.trace_id,
+ surface=trace.surface,
+ )
+ return results
+
+
+@with_team_scope(canonical=True)
+def _search_knowledge(
+ team_id: int,
+ query: str,
+ *,
+ limit: int = 10,
+ use_semantic: bool = False,
+ query_embedding: list[float] | None = None,
+ library_id: UUID | None = None,
) -> list[KnowledgeSearchResult]:
"""
Hybrid (lexical + semantic) relevance search over BK chunks.
@@ -2884,6 +2931,7 @@ def search_knowledge_for_team(
if library_id is not None:
with team_scope(canonical_team_id, canonical=True):
if not _library_has_sources(canonical_team_id, library_id):
+ billing.record_query(team_id=canonical_team_id, trace_id=trace.trace_id, surface=trace.surface)
return []
embedding: list[float] | None = None
try:
@@ -2899,6 +2947,7 @@ def search_knowledge_for_team(
use_semantic=embedding is not None,
query_embedding=embedding,
library_id=library_id,
+ trace=trace,
)
@@ -2910,6 +2959,8 @@ async def async_search_knowledge_for_team(
limit: int = 10,
) -> list[KnowledgeSearchResult]:
"""Async hybrid search. Embedding failure or timeout falls back to full-text search."""
+ # The search stays on `team.id` because callers only authorize `team`; usage belongs to the parent.
+ usage_team_id = await database_sync_to_async(resolve_effective_team_id, thread_sensitive=False)(team.id)
embedding: list[float] | None = None
try:
response = await asyncio.wait_for(
@@ -2926,6 +2977,8 @@ async def async_search_knowledge_for_team(
limit=limit,
use_semantic=embedding is not None,
query_embedding=embedding,
+ trace=trace,
+ usage_team_id=usage_team_id,
)
diff --git a/products/business_knowledge/backend/migrations/0025_knowledge_usage_record.py b/products/business_knowledge/backend/migrations/0025_knowledge_usage_record.py
new file mode 100644
index 000000000000..2b0e854bd3ee
--- /dev/null
+++ b/products/business_knowledge/backend/migrations/0025_knowledge_usage_record.py
@@ -0,0 +1,52 @@
+from django.db import migrations, models
+
+import posthog.uuidt
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("business_knowledge", "0024_knowledge_libraries"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="KnowledgeUsageRecord",
+ fields=[
+ (
+ "id",
+ models.UUIDField(default=posthog.uuidt.uuid7, editable=False, primary_key=True, serialize=False),
+ ),
+ ("team_id", models.BigIntegerField()),
+ (
+ "usage_key",
+ models.CharField(
+ choices=[
+ ("business_knowledge_chunks", "Chunks ingested"),
+ ("business_knowledge_queries", "Queries"),
+ ],
+ max_length=64,
+ ),
+ ),
+ ("quantity", models.PositiveIntegerField()),
+ ("surface", models.CharField(blank=True, default="", max_length=32)),
+ ("idempotency_key", models.CharField(blank=True, max_length=128, null=True)),
+ ("source_id", models.UUIDField(blank=True, null=True)),
+ ("document_id", models.UUIDField(blank=True, null=True)),
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ],
+ options={
+ "db_table": "posthog_business_knowledge_knowledgeusagerecord",
+ "indexes": [
+ models.Index(fields=["created_at", "team_id"], name="bk_usage_created_team"),
+ models.Index(fields=["team_id", "usage_key", "created_at"], name="bk_usage_team_key_created"),
+ ],
+ "constraints": [
+ models.UniqueConstraint(
+ condition=models.Q(("idempotency_key__isnull", False)),
+ fields=("usage_key", "idempotency_key"),
+ name="bk_usage_idempotency_key",
+ )
+ ],
+ },
+ ),
+ ]
diff --git a/products/business_knowledge/backend/migrations/max_migration.txt b/products/business_knowledge/backend/migrations/max_migration.txt
index 41cc182df651..8e9da83fb846 100644
--- a/products/business_knowledge/backend/migrations/max_migration.txt
+++ b/products/business_knowledge/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0024_knowledge_libraries
+0025_knowledge_usage_record
diff --git a/products/business_knowledge/backend/models/__init__.py b/products/business_knowledge/backend/models/__init__.py
index 69244e1ddba1..f2012a8b248a 100644
--- a/products/business_knowledge/backend/models/__init__.py
+++ b/products/business_knowledge/backend/models/__init__.py
@@ -21,6 +21,7 @@ from .knowledge_library import KnowledgeLibrary
from .knowledge_library_repository import KnowledgeLibraryRepository
from .knowledge_library_source import KnowledgeLibrarySource
from .knowledge_source import KnowledgeSource
+from .knowledge_usage_record import KnowledgeUsageKey, KnowledgeUsageRecord
from .playground_chat import PlaygroundChat
from .playground_turn import PlaygroundTurn
from .team_business_knowledge_config import TeamBusinessKnowledgeConfig
@@ -39,6 +40,8 @@ __all__ = [
"KnowledgeLibraryRepository",
"KnowledgeLibrarySource",
"KnowledgeSource",
+ "KnowledgeUsageKey",
+ "KnowledgeUsageRecord",
"LearningProvider",
"PlaygroundChat",
"PlaygroundTurn",
diff --git a/products/business_knowledge/backend/models/knowledge_usage_record.py b/products/business_knowledge/backend/models/knowledge_usage_record.py
new file mode 100644
index 000000000000..e9e11fa20f8f
--- /dev/null
+++ b/products/business_knowledge/backend/models/knowledge_usage_record.py
@@ -0,0 +1,54 @@
+from django.db import models
+
+from posthog.models.scoping.manager import TeamScopedManager
+from posthog.models.utils import UUIDModel
+
+
+class KnowledgeUsageKey(models.TextChoices):
+ """Billable meters. They must match the `usage_key` values in the billing service's plans config."""
+
+ CHUNKS = "business_knowledge_chunks", "Chunks ingested"
+ QUERIES = "business_knowledge_queries", "Queries"
+
+
+class KnowledgeUsageRecord(UUIDModel):
+ """Append-only record of one unit of billable business knowledge work.
+
+ Ingestion writes one row per chunk write, in the same transaction as the chunks, so a rolled-back
+ write records nothing. Retrieval writes one row per search. Rows are never updated, and deleting the
+ source, document or chunks they describe leaves them in place, so a delete never removes usage.
+
+ Nothing reads this table for billing yet. It meters usage so the pricing can be checked against real
+ data before the usage report and the billing service start counting it.
+ """
+
+ # Canonical team, no FK: deleting a team must not cascade into usage history.
+ team_id = models.BigIntegerField()
+ usage_key = models.CharField(max_length=64, choices=KnowledgeUsageKey.choices)
+ quantity = models.PositiveIntegerField()
+ surface = models.CharField(max_length=32, blank=True, default="")
+ # The search's trace id, so a retried Temporal activity is counted once.
+ idempotency_key = models.CharField(max_length=128, null=True, blank=True)
+ # Plain ids, kept to answer "why was I billed for this?" after a delete.
+ source_id = models.UUIDField(null=True, blank=True)
+ document_id = models.UUIDField(null=True, blank=True)
+ created_at = models.DateTimeField(auto_now_add=True)
+
+ objects = TeamScopedManager()
+
+ class Meta:
+ db_table = "posthog_business_knowledge_knowledgeusagerecord"
+ constraints = [
+ models.UniqueConstraint(
+ fields=["usage_key", "idempotency_key"],
+ condition=models.Q(idempotency_key__isnull=False),
+ name="bk_usage_idempotency_key",
+ ),
+ ]
+ indexes = [
+ models.Index(fields=["created_at", "team_id"], name="bk_usage_created_team"),
+ models.Index(fields=["team_id", "usage_key", "created_at"], name="bk_usage_team_key_created"),
+ ]
+
+ def __str__(self) -> str:
+ return f"{self.usage_key} x{self.quantity} (team {self.team_id})"
diff --git a/products/business_knowledge/backend/temporal/learning/activities/analyze.py b/products/business_knowledge/backend/temporal/learning/activities/analyze.py
index 7d0f425d84f5..1716993bb49e 100644
--- a/products/business_knowledge/backend/temporal/learning/activities/analyze.py
+++ b/products/business_knowledge/backend/temporal/learning/activities/analyze.py
@@ -27,6 +27,7 @@ from products.business_knowledge.backend import learning_settings, logic
from products.business_knowledge.backend.constants import BK_EMBEDDING_MODEL, BK_QUERY_EMBEDDING_TIMEOUT
from products.business_knowledge.backend.learning.contracts import EvidenceBundle, EvidenceRef
from products.business_knowledge.backend.learning.providers import get_learning_provider
+from products.business_knowledge.backend.llm_telemetry import RetrievalTrace, stable_trace_id
from products.business_knowledge.backend.models import KnowledgeLearningRun, LearningRunResult, LearningRunStatus
from ..constants import (
@@ -475,7 +476,7 @@ def _contradiction_gate_rejection(
return "none"
-def _search_existing_knowledge(team: Team, query: str) -> list[logic.KnowledgeSearchResult]:
+def _search_existing_knowledge(team: Team, query: str, *, run_id: UUID) -> list[logic.KnowledgeSearchResult]:
embedding = generate_embedding(
team,
query,
@@ -490,6 +491,8 @@ def _search_existing_knowledge(team: Team, query: str) -> list[logic.KnowledgeSe
limit=LEARNING_SEARCH_LIMIT,
use_semantic=True,
query_embedding=embedding,
+ # Keyed on the run, so a retried analysis is counted once.
+ trace=RetrievalTrace(surface="learning", trace_id=stable_trace_id("learning", run_id)),
)
@@ -762,7 +765,7 @@ def _analyze(run: KnowledgeLearningRun, input: AnalyzeLearningEvidenceInput) ->
return _finish_without_knowledge(run, rejection_code="pii")
try:
- results = _search_existing_knowledge(run.team, generated_text)
+ results = _search_existing_knowledge(run.team, generated_text, run_id=run.id)
except Exception:
raise LearningAnalysisError("knowledge_search_failed") from None
retrieved = _render_search_context(results)
diff --git a/products/business_knowledge/backend/tests/test_billing.py b/products/business_knowledge/backend/tests/test_billing.py
new file mode 100644
index 000000000000..c432de45bed6
--- /dev/null
+++ b/products/business_knowledge/backend/tests/test_billing.py
@@ -0,0 +1,188 @@
+import asyncio
+from uuid import UUID
+
+from posthog.test.base import BaseTest
+from unittest.mock import patch
+
+from django.db import transaction
+from django.utils import timezone
+
+from posthog.models.scoping import team_scope
+from posthog.models.team import Team
+
+from products.business_knowledge.backend import billing, logic
+from products.business_knowledge.backend.llm_telemetry import RetrievalTrace, stable_trace_id
+from products.business_knowledge.backend.models import (
+ KnowledgeChunk,
+ KnowledgeLibrary,
+ KnowledgeUsageKey,
+ KnowledgeUsageRecord,
+)
+
+TEXT = "Refunds are available within 30 days.\n\nShipping is free above 50 euros."
+
+
+class _UsageTest(BaseTest):
+ def _rows(self, usage_key: KnowledgeUsageKey, team_id: int | None = None) -> list[KnowledgeUsageRecord]:
+ return list(
+ KnowledgeUsageRecord.objects.unscoped()
+ .filter(team_id=team_id or self.team.id, usage_key=usage_key)
+ .order_by("created_at")
+ )
+
+
+class TestIngestionMetering(_UsageTest):
+ def test_text_source_meters_every_chunk_it_writes(self) -> None:
+ source = logic.create_text_source(team_id=self.team.id, created_by_id=self.user.id, name="Policies", text=TEXT)
+
+ chunks = KnowledgeChunk.objects.unscoped().filter(source_id=source.id).count()
+ assert chunks > 0
+ assert [(r.quantity, r.surface, r.source_id) for r in self._rows(KnowledgeUsageKey.CHUNKS)] == [
+ (chunks, "text", source.id)
+ ]
+
+ def test_editing_text_meters_the_new_chunks_again(self) -> None:
+ source = logic.create_text_source(team_id=self.team.id, created_by_id=self.user.id, name="Policies", text=TEXT)
+
+ logic.update_text_source(source_id=source.id, team_id=self.team.id, name=None, text=TEXT + "\n\nUpdated.")
+
+ assert len(self._rows(KnowledgeUsageKey.CHUNKS)) == 2
+
+ def test_learned_source_meters_as_generated_ingestion(self) -> None:
+ result = logic.create_generated_knowledge_document(
+ logic.CreateGeneratedKnowledgeDocument(
+ team_id=self.team.id,
+ provider="conversations",
+ ticket_id=UUID("10000000-0000-0000-0000-000000000001"),
+ ticket_number=42,
+ source_team_id=self.team.id,
+ resolution_comment_id=UUID("20000000-0000-0000-0000-000000000002"),
+ analysis_version="post_resolution_v1",
+ title="Refund policy",
+ content="Refunds are available within 30 days.",
+ evidence_revision_at=timezone.now(),
+ )
+ )
+
+ rows = self._rows(KnowledgeUsageKey.CHUNKS)
+ assert [(r.surface, r.source_id) for r in rows] == [("generated", result.source_id)]
+
+ def test_a_rolled_back_write_meters_nothing(self) -> None:
+ with self.assertRaises(RuntimeError), transaction.atomic():
+ billing.record_chunks_ingested(
+ team_id=self.team.id, quantity=3, surface="text", source_id=None, document_id=None
+ )
+ raise RuntimeError("rolled back")
+
+ assert self._rows(KnowledgeUsageKey.CHUNKS) == []
+
+ def test_zero_chunks_write_no_row(self) -> None:
+ billing.record_chunks_ingested(
+ team_id=self.team.id, quantity=0, surface="text", source_id=None, document_id=None
+ )
+
+ assert self._rows(KnowledgeUsageKey.CHUNKS) == []
+
+ def test_a_metering_failure_does_not_fail_the_ingest(self) -> None:
+ with patch.object(KnowledgeUsageRecord.objects, "unscoped", side_effect=RuntimeError("ledger down")):
+ source = logic.create_text_source(
+ team_id=self.team.id, created_by_id=self.user.id, name="Policies", text=TEXT
+ )
+
+ assert KnowledgeChunk.objects.unscoped().filter(source_id=source.id).exists()
+ assert self._rows(KnowledgeUsageKey.CHUNKS) == []
+
+
+class TestRetrievalMetering(_UsageTest):
+ def test_a_search_with_a_trace_meters_one_query(self) -> None:
+ trace = RetrievalTrace(surface="api")
+
+ logic.search_knowledge(self.team.id, "refunds", trace=trace)
+
+ assert [(r.quantity, r.surface, r.idempotency_key) for r in self._rows(KnowledgeUsageKey.QUERIES)] == [
+ (1, "api", trace.trace_id)
+ ]
+
+ def test_a_search_without_a_trace_is_not_metered(self) -> None:
+ logic.search_knowledge(self.team.id, "refunds")
+
+ assert self._rows(KnowledgeUsageKey.QUERIES) == []
+
+ def test_the_internal_docs_shadow_search_is_not_metered(self) -> None:
+ logic.search_knowledge(self.team.id, "refunds", trace=RetrievalTrace(surface="docs_shadow"))
+
+ assert self._rows(KnowledgeUsageKey.QUERIES) == []
+
+ def test_a_retried_search_with_the_same_trace_meters_once(self) -> None:
+ trace = RetrievalTrace(surface="support", trace_id=stable_trace_id("workflow", "run", "activity", 0, "refunds"))
+
+ logic.search_knowledge(self.team.id, "refunds", trace=trace)
+ logic.search_knowledge(self.team.id, "refunds", trace=trace)
+
+ assert len(self._rows(KnowledgeUsageKey.QUERIES)) == 1
+
+ def test_a_search_against_an_empty_library_is_metered(self) -> None:
+ with team_scope(self.team.id, canonical=True):
+ library = KnowledgeLibrary.objects.create(team_id=self.team.id, name="Empty")
+ trace = RetrievalTrace(surface="api")
+
+ results = logic.search_knowledge_for_team(self.team, "refunds", trace=trace, library_id=library.id)
+
+ assert results == []
+ assert [r.idempotency_key for r in self._rows(KnowledgeUsageKey.QUERIES)] == [trace.trace_id]
+
+ def test_a_search_that_raises_meters_nothing(self) -> None:
+ with (
+ patch.object(logic, "_search_knowledge", side_effect=RuntimeError("search failed")),
+ self.assertRaises(RuntimeError),
+ ):
+ logic.search_knowledge(self.team.id, "refunds", trace=RetrievalTrace(surface="api"))
+
+ assert self._rows(KnowledgeUsageKey.QUERIES) == []
+
+ def test_a_metering_failure_does_not_fail_the_search(self) -> None:
+ with patch.object(KnowledgeUsageRecord.objects, "unscoped", side_effect=RuntimeError("ledger down")):
+ results = logic.search_knowledge(self.team.id, "refunds", trace=RetrievalTrace(surface="api"))
+
+ assert results == []
+
+ def test_sync_search_from_a_child_environment_meters_the_parent(self) -> None:
+ child = Team.objects.create(
+ organization=self.organization, parent_team=self.team, project=self.team.project, name="Child environment"
+ )
+
+ with patch.object(logic, "generate_embedding", side_effect=RuntimeError("no embeddings")):
+ logic.search_knowledge_for_team(child, "refunds", trace=RetrievalTrace(surface="api"))
+
+ assert len(self._rows(KnowledgeUsageKey.QUERIES)) == 1
+ assert self._rows(KnowledgeUsageKey.QUERIES, team_id=child.id) == []
+
+ def test_async_search_from_a_child_environment_searches_the_child_and_meters_the_parent(self) -> None:
+ # PostHog AI only authorizes the environment it runs in, so the search must not widen to the parent.
+ child = Team.objects.create(
+ organization=self.organization, parent_team=self.team, project=self.team.project, name="Child environment"
+ )
+
+ def run_inline(fn, **_kwargs):
+ async def call(*args, **kwargs):
+ return fn(*args, **kwargs)
+
+ return call
+
+ with (
+ patch.object(logic, "database_sync_to_async", side_effect=run_inline),
+ patch.object(logic, "resolve_effective_team_id", return_value=self.team.id),
+ patch.object(logic, "async_generate_embedding", side_effect=RuntimeError("no embeddings")),
+ patch.object(logic, "_search_knowledge", return_value=[]) as search,
+ patch.object(billing, "record_query") as record_query,
+ ):
+ asyncio.run(
+ logic.async_search_knowledge_for_team(child, "refunds", trace=RetrievalTrace(surface="posthog_ai"))
+ )
+
+ assert search.call_args.args[0] == child.id
+ assert record_query.call_args.kwargs["team_id"] == self.team.id
+
+ def test_stable_trace_ids_repeat_for_the_same_parts_only(self) -> None:
+ assert stable_trace_id("a", 1) == stable_trace_id("a", 1)
+ assert stable_trace_id("a", 1) != stable_trace_id("a", 2)
diff --git a/products/business_knowledge/backend/tests/test_learning_analyzer.py b/products/business_knowledge/backend/tests/test_learning_analyzer.py
index 8aece4038ff3..c27c0a9171c2 100644
--- a/products/business_knowledge/backend/tests/test_learning_analyzer.py
+++ b/products/business_knowledge/backend/tests/test_learning_analyzer.py
@@ -19,6 +19,7 @@ from posthog.models.team import Team
from products.business_knowledge.backend import learning_settings, logic
from products.business_knowledge.backend.learning.contracts import EvidenceBundle, EvidenceRef, evidence_key_for
+from products.business_knowledge.backend.llm_telemetry import RetrievalTrace, stable_trace_id
from products.business_knowledge.backend.models import (
KnowledgeDocument,
KnowledgeLearningRun,
@@ -455,6 +456,9 @@ class TestLearningAnalyzer:
]
assert search.call_args.kwargs["use_semantic"] is True
assert search.call_args.kwargs["query_embedding"] == [0.1, 0.2]
+ assert search.call_args.kwargs["trace"] == RetrievalTrace(
+ surface="learning", trace_id=stable_trace_id("learning", run.id)
+ )
publish.assert_not_called()
@pytest.mark.parametrize(
diff --git a/products/conversations/backend/temporal/ai_reply/activities/retrieve.py b/products/conversations/backend/temporal/ai_reply/activities/retrieve.py
index fdb059d26a3b..e63e31376580 100644
--- a/products/conversations/backend/temporal/ai_reply/activities/retrieve.py
+++ b/products/conversations/backend/temporal/ai_reply/activities/retrieve.py
@@ -17,6 +17,7 @@ from products.business_knowledge.backend.logic import (
get_document_window,
rerank_chunks,
search_knowledge_for_team,
+ stable_trace_id,
)
from products.business_knowledge.backend.models import KnowledgeChunk
from products.conversations.backend.temporal.ai_reply.constants import (
@@ -34,17 +35,20 @@ logger = structlog.get_logger(__name__)
@close_db_connections
async def support_retrieve_activity(input: RetrieveInput) -> RetrieveOutput:
"""Search BK + rerank. On widen attempts, also fetch document windows around prior citations."""
+ info = activity.info()
+ # Stable across retries of this activity, so a retried search is counted once.
+ trace_seed = f"{info.workflow_id}:{info.workflow_run_id}:{info.activity_id}"
async with Heartbeater():
- return await database_sync_to_async(_retrieve_sync, thread_sensitive=False)(input)
+ return await database_sync_to_async(_retrieve_sync, thread_sensitive=False)(input, trace_seed)
-def _retrieve_sync(input: RetrieveInput) -> RetrieveOutput:
+def _retrieve_sync(input: RetrieveInput, trace_seed: str) -> RetrieveOutput:
team = Team.objects.select_related("organization").get(id=input.team_id)
all_results = []
seen_chunk_ids: set[str] = set()
- for query in input.queries:
- trace = RetrievalTrace(surface="support")
+ for index, query in enumerate(input.queries):
+ trace = RetrievalTrace(surface="support", trace_id=stable_trace_id(trace_seed, index, query))
results = search_knowledge_for_team(team, query, limit=RETRIEVE_LIMIT, trace=trace)
reranked = rerank_chunks(team, query, results, top_k=RERANK_TOP_K, trace=trace)
for r in reranked:
diff --git a/products/conversations/backend/temporal/tests/test_retrieve_metering.py b/products/conversations/backend/temporal/tests/test_retrieve_metering.py
new file mode 100644
index 000000000000..8f61f9376e01
--- /dev/null
+++ b/products/conversations/backend/temporal/tests/test_retrieve_metering.py
@@ -0,0 +1,32 @@
+from posthog.test.base import BaseTest
+from unittest.mock import patch
+
+from products.business_knowledge.backend import logic
+from products.business_knowledge.backend.models import KnowledgeUsageKey, KnowledgeUsageRecord
+from products.conversations.backend.temporal.ai_reply.activities.retrieve import _retrieve_sync
+from products.conversations.backend.temporal.ai_reply.schemas import RetrieveInput
+
+RETRIEVE_MODULE = "products.conversations.backend.temporal.ai_reply.activities.retrieve"
+
+
+class TestSupportRetrieveMetering(BaseTest):
+ def _query_count(self) -> int:
+ return (
+ KnowledgeUsageRecord.objects.unscoped()
+ .filter(team_id=self.team.id, usage_key=KnowledgeUsageKey.QUERIES)
+ .count()
+ )
+
+ def test_each_query_is_metered_once_per_activity_even_when_retried(self) -> None:
+ input = RetrieveInput(team_id=self.team.id, queries=["refunds", "shipping"])
+
+ with (
+ patch(f"{RETRIEVE_MODULE}.rerank_chunks", return_value=[]),
+ patch.object(logic, "generate_embedding", side_effect=RuntimeError("no embeddings")),
+ ):
+ _retrieve_sync(input, "workflow:run-1:activity-1")
+ _retrieve_sync(input, "workflow:run-1:activity-1")
+ assert self._query_count() == 2
+
+ _retrieve_sync(input, "workflow:run-2:activity-1")
+ assert self._query_count() == 4