:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(business-knowledge): route search rerank through the ai gateway (#123444)
Jonathan Mielooe7b16e9271ac
diff --git a/products/business_knowledge/backend/constants.py b/products/business_knowledge/backend/constants.pyindex 058b6765a72d..11cd9073f09c 100644--- a/products/business_knowledge/backend/constants.py+++ b/products/business_knowledge/backend/constants.py@@ -4,6 +4,8 @@ Constants and tunables for business_knowledge.import datetime+import httpx+# Per-team caps. Enforced in the create endpoint, not at the DB layer — easier# to relax for a single paying customer without a migration.MAX_SOURCES_PER_TEAM = 500@@ -191,6 +193,7 @@ BK_RRF_SCORE_FLOOR = 0.015BK_RERANK_MODEL = "claude-haiku-4-5"# Max chars of chunk content included in the rerank prompt per candidate.BK_RERANK_SNIPPET_CHARS = 500+BK_RERANK_TIMEOUT = httpx.Timeout(30.0, connect=5.0)# Timeout (seconds) for the async embedding call on the query path. If the# embedding service is slow, FTS alone fires (graceful degradation).BK_QUERY_EMBEDDING_TIMEOUT = 5.0diff --git a/products/business_knowledge/backend/llm_telemetry.py b/products/business_knowledge/backend/llm_telemetry.pyindex d65228e5ec80..a2ace172d719 100644--- a/products/business_knowledge/backend/llm_telemetry.py+++ b/products/business_knowledge/backend/llm_telemetry.py@@ -10,12 +10,11 @@ trace, so cost can be summed per search.import mathimport dataclassesfrom collections.abc import Iterable, Mapping-from typing import Any, Literal+from typing import Literalfrom uuid import UUID, uuid4import structlogimport posthoganalytics-from posthoganalytics.ai.langchain.callbacks import CallbackHandlerfrom posthog.dataclasses import frozenfrom posthog.llm.gateway_client import team_distinct_id@@ -93,60 +92,3 @@ def capture_embedding(*, team_id: int, input_tokens: int, trace_id: str, propert)except Exception:logger.warning("business_knowledge.embedding_cost_capture_failed", team_id=team_id, exc_info=True)---class _QuietCallbackHandler(CallbackHandler):- """- Records successful generations only. On a model error the base handler- records the error message and calls ``capture_exception``. That message can- contain prompt or output text, which privacy mode does not redact, and every- caller here already handles the failure itself. The base handler also logs- every prompt at DEBUG level, which privacy mode does not redact either.- """-- def _log_debug_event(- self,- event_name: str,- run_id: UUID,- parent_run_id: UUID | None = None,- **kwargs: Any,- ) -> None:- _ = event_name, run_id, parent_run_id, kwargs-- def on_llm_error(- self,- error: BaseException,- *,- run_id: UUID,- parent_run_id: UUID | None = None,- **kwargs: Any,- ) -> None:- _ = error, run_id, parent_run_id, kwargs-- def on_chain_error(- self,- error: BaseException,- *,- run_id: UUID,- parent_run_id: UUID | None = None,- **kwargs: Any,- ) -> None:- _ = error, run_id, parent_run_id, kwargs---def trace_callback(team_id: int, *, trace_id: str, properties: Mapping[str, str | int]) -> CallbackHandler | None:- """LangChain callback that captures ``$ai_generation`` for a model call made outside a PostHog AI graph."""- # default_client stays None until the first module-level capture.- client = posthoganalytics.default_client- if client is None and not posthoganalytics.disabled:- client = posthoganalytics.setup()- if client is None:- return None- return _QuietCallbackHandler(- client,- distinct_id=team_distinct_id(team_id),- trace_id=trace_id,- # Prompts and outputs contain customer content, so only token counts reach analytics.- privacy_mode=True,- properties=dict(properties),- )diff --git a/products/business_knowledge/backend/logic.py b/products/business_knowledge/backend/logic.pyindex 771848c59674..1a004ac93f0d 100644--- a/products/business_knowledge/backend/logic.py+++ b/products/business_knowledge/backend/logic.py@@ -43,22 +43,18 @@ from django.db.models.functions import Cast, Coalesce, Substrfrom django.utils import timezoneimport structlog-from langchain_core.messages import HumanMessage, SystemMessagefrom posthog.api.embedding_worker import EmbeddingResponse, async_generate_embedding, generate_embeddingfrom posthog.dataclasses import frozenfrom posthog.helpers.full_text_search import process_query-from posthog.models.organization import OrganizationMembership+from posthog.llm.gateway_client import build_ai_gateway_anthropic_client, team_distinct_idfrom posthog.models.scoping import team_scope, with_team_scopefrom posthog.models.scoping.manager import resolve_effective_team_idfrom posthog.models.team.team import Team-from posthog.models.user import Userfrom posthog.ph_client import feature_enabled_or_falsefrom posthog.security.url_validation import is_url_allowedfrom posthog.sync import database_sync_to_async-from ee.hogai.llm import MaxChatAnthropic-from . import crawl, discover, file_parse, html_parse, llm_telemetry, url_fetchfrom .constants import (BK_DRILLDOWN_DEFAULT_RADIUS,@@ -71,6 +67,7 @@ from .constants import (BK_QUERY_EMBEDDING_TIMEOUT,BK_RERANK_MODEL,BK_RERANK_SNIPPET_CHARS,+ BK_RERANK_TIMEOUT,BK_RRF_K,BK_RRF_SCORE_FLOOR,BK_SEARCH_MAX_LIMIT,@@ -2935,18 +2932,6 @@ Return ONLY chunk_id UUIDs in order from most to least relevant, one UUID per liDo not include any other text."""-def _resolve_active_org_user(team: Team) -> User:- membership = (- OrganizationMembership.objects.select_related("user")- .filter(organization=team.organization, user__is_active=True)- .order_by("id")- .first()- )- if not membership:- raise RuntimeError(f"No active users in organization '{team.organization.name}' (team {team.id})")- return membership.user--def _build_rerank_user_prompt(query: str, results: list[KnowledgeSearchResult]) -> str:lines = [f"Query: {query}", "", "Candidates:"]for index, result in enumerate(results, start=1):@@ -3000,34 +2985,25 @@ def rerank_chunks(valid_ids = {result.chunk_id for result in results}id_to_result = {result.chunk_id: result for result in results}- properties = llm_telemetry.retrieval_properties(- team_id=team.id, trace=trace, feature=llm_telemetry.RETRIEVAL_RERANK_FEATURE- )try:- user = _resolve_active_org_user(team)- llm = MaxChatAnthropic(- model=BK_RERANK_MODEL,- streaming=False,- user=user,- team=team,- max_tokens=1024,- billable=False,- inject_context=False,- posthog_properties=properties,- )- # MaxChatAnthropic captures no $ai_generation by itself. The cost reaches analytics only through this callback.- callback = llm_telemetry.trace_callback(team.id, trace_id=trace.trace_id, properties=properties)- response = llm.invoke(- [- SystemMessage(content=_RERANK_SYSTEM_PROMPT),- HumanMessage(content=_build_rerank_user_prompt(query, results)),- ],- config={"callbacks": [callback]} if callback is not None else None,+ client = build_ai_gateway_anthropic_client(+ ai_product=llm_telemetry.BK_AI_PRODUCT,+ trace_id=trace.trace_id,+ distinct_id=team_distinct_id(team.id),+ team_id=team.id,+ properties={"ai_feature": llm_telemetry.RETRIEVAL_RERANK_FEATURE, "surface": trace.surface},)- content = response.content- if isinstance(content, list):- content = "".join(str(item) for item in content)- ranked_ids = _parse_reranked_chunk_ids(str(content), valid_ids)+ with client.with_options(timeout=BK_RERANK_TIMEOUT, max_retries=3) as bounded:+ response = bounded.messages.create(+ model=BK_RERANK_MODEL,+ max_tokens=1024,+ system=_RERANK_SYSTEM_PROMPT,+ messages=[{"role": "user", "content": _build_rerank_user_prompt(query, results)}],+ # Prompts and outputs contain customer content, so only token counts reach analytics.+ extra_headers={"X-PostHog-Billable": "false", "X-PostHog-Privacy-Mode": "true"},+ )+ content = "".join(block.text for block in response.content if block.type == "text")+ ranked_ids = _parse_reranked_chunk_ids(content, valid_ids)if ranked_ids is None:return original_order[:top_k]diff --git a/products/business_knowledge/backend/tests/test_rerank_chunks.py b/products/business_knowledge/backend/tests/test_rerank_chunks.pyindex 8491b61d1b7f..92dd4b2db745 100644--- a/products/business_knowledge/backend/tests/test_rerank_chunks.py+++ b/products/business_knowledge/backend/tests/test_rerank_chunks.py@@ -3,13 +3,36 @@ from uuid import UUID, uuid4from posthog.test.base import BaseTestfrom unittest.mock import MagicMock, patch-from langchain_core.messages import AIMessage+from anthropic.types import Message, TextBlock, Usagefrom parameterized import parameterized+from products.business_knowledge.backend.constants import BK_RERANK_TIMEOUTfrom products.business_knowledge.backend.llm_telemetry import RetrievalTracefrom products.business_knowledge.backend.logic import KnowledgeSearchResult, rerank_chunks+def _model_reply(text: str) -> Message:+ return Message(+ id="msg_test",+ type="message",+ role="assistant",+ model="claude-test",+ content=[TextBlock(type="text", text=text)],+ stop_reason="end_turn",+ stop_sequence=None,+ usage=Usage(input_tokens=1, output_tokens=1),+ )+++def _gateway_client(reply: Message) -> MagicMock:+ client = MagicMock()+ client.with_options.return_value.__enter__.return_value.messages.create.return_value = reply+ return client+++_BUILD_CLIENT = "products.business_knowledge.backend.logic.build_ai_gateway_anthropic_client"++class TestRerankChunks(BaseTest):def _make_result(self, content: str, *, chunk_id: UUID | None = None) -> KnowledgeSearchResult:return KnowledgeSearchResult(@@ -30,10 +53,9 @@ class TestRerankChunks(BaseTest):def test_respects_top_k(self) -> None:results = [self._make_result(f"content {index}") for index in range(5)]reordered_ids = [results[2].chunk_id, results[0].chunk_id, results[4].chunk_id]- mock_llm = MagicMock()- mock_llm.invoke.return_value = AIMessage(content="\n".join(str(chunk_id) for chunk_id in reordered_ids))+ client = _gateway_client(_model_reply("\n".join(str(chunk_id) for chunk_id in reordered_ids)))- with patch("products.business_knowledge.backend.logic.MaxChatAnthropic", return_value=mock_llm):+ with patch(_BUILD_CLIENT, return_value=client):reranked = rerank_chunks(self.team, "query", results, top_k=2, trace=RetrievalTrace(surface="api"))assert len(reranked) == 2@@ -42,7 +64,7 @@ class TestRerankChunks(BaseTest):@parameterized.expand([("parse_failure", 3, "invalid_response"),- ("model_failure", 2, "init_error"),+ ("gateway_not_configured", 2, "init_error"),])def test_rrf_fallback_on_llm_failure(self, _name: str, num_results: int, failure_mode: str) -> None:@@ -50,13 +72,11 @@ class TestRerankChunks(BaseTest):top_k = 2if failure_mode == "invalid_response":- mock_llm = MagicMock()- mock_llm.invoke.return_value = AIMessage(content="not valid uuids here")- patch_kwargs: dict = {"return_value": mock_llm}+ patch_kwargs: dict = {"return_value": _gateway_client(_model_reply("not valid uuids here"))}else:- patch_kwargs = {"side_effect": RuntimeError("llm unavailable")}+ patch_kwargs = {"side_effect": ValueError("AI_GATEWAY_URL is not set")}- with patch("products.business_knowledge.backend.logic.MaxChatAnthropic", **patch_kwargs):+ with patch(_BUILD_CLIENT, **patch_kwargs):reranked = rerank_chunks(self.team, "query", results, top_k=top_k, trace=RetrievalTrace(surface="api"))assert [result.chunk_id for result in reranked] == [result.chunk_id for result in results[:top_k]]@@ -66,26 +86,24 @@ class TestRerankChunks(BaseTest):second = self._make_result("pricing tiers")third = self._make_result("deployment guide")results = [first, second, third]- mock_llm = MagicMock()- mock_llm.invoke.return_value = AIMessage(content=f"{third.chunk_id}\n{first.chunk_id}\n{second.chunk_id}")+ client = _gateway_client(_model_reply(f"{third.chunk_id}\n{first.chunk_id}\n{second.chunk_id}"))trace = RetrievalTrace(surface="support")- with (- patch("products.business_knowledge.backend.logic.MaxChatAnthropic", return_value=mock_llm),- patch(- "products.business_knowledge.backend.llm_telemetry.posthoganalytics",- MagicMock(default_client=MagicMock(), disabled=False),- ),- ):+ with patch(_BUILD_CLIENT, return_value=client) as build_client:reranked = rerank_chunks(self.team, "how do refunds work", results, top_k=3, trace=trace)assert [result.chunk_id for result in reranked] == [third.chunk_id, first.chunk_id, second.chunk_id]- callback = mock_llm.invoke.call_args.kwargs["config"]["callbacks"][0]- assert callback._trace_id == trace.trace_id- assert callback._privacy_mode is True- assert callback._properties == {- "ai_product": "business_knowledge",+ assert build_client.call_args.kwargs["trace_id"] == trace.trace_id+ assert build_client.call_args.kwargs["ai_product"] == "business_knowledge"+ assert build_client.call_args.kwargs["team_id"] == self.team.id+ assert build_client.call_args.kwargs["properties"] == {"ai_feature": "bk_retrieval_rerank",- "team_id": self.team.id,"surface": "support",}+ assert client.with_options.call_args.kwargs == {"timeout": BK_RERANK_TIMEOUT, "max_retries": 3}+ assert client.with_options.return_value.__enter__.return_value.messages.create.call_args.kwargs[+ "extra_headers"+ ] == {+ "X-PostHog-Billable": "false",+ "X-PostHog-Privacy-Mode": "true",+ }