Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(business-knowledge): route search rerank through the ai gateway (#123444)

Jonathan Mielooe7b16e9271ac
diff --git a/products/business_knowledge/backend/constants.py b/products/business_knowledge/backend/constants.py
index 058b6765a72d..11cd9073f09c 100644
--- a/products/business_knowledge/backend/constants.py
+++ b/products/business_knowledge/backend/constants.py
@@ -4,6 +4,8 @@ Constants and tunables for business_knowledge.
import datetime
+import httpx
+
# Per-team caps. Enforced in the create endpoint, not at the DB layer — easier
# to relax for a single paying customer without a migration.
MAX_SOURCES_PER_TEAM = 500
@@ -191,6 +193,7 @@ BK_RRF_SCORE_FLOOR = 0.015
BK_RERANK_MODEL = "claude-haiku-4-5"
# Max chars of chunk content included in the rerank prompt per candidate.
BK_RERANK_SNIPPET_CHARS = 500
+BK_RERANK_TIMEOUT = httpx.Timeout(30.0, connect=5.0)
# Timeout (seconds) for the async embedding call on the query path. If the
# embedding service is slow, FTS alone fires (graceful degradation).
BK_QUERY_EMBEDDING_TIMEOUT = 5.0
diff --git a/products/business_knowledge/backend/llm_telemetry.py b/products/business_knowledge/backend/llm_telemetry.py
index d65228e5ec80..a2ace172d719 100644
--- a/products/business_knowledge/backend/llm_telemetry.py
+++ b/products/business_knowledge/backend/llm_telemetry.py
@@ -10,12 +10,11 @@ trace, so cost can be summed per search.
import math
import dataclasses
from collections.abc import Iterable, Mapping
-from typing import Any, Literal
+from typing import Literal
from uuid import UUID, uuid4
import structlog
import posthoganalytics
-from posthoganalytics.ai.langchain.callbacks import CallbackHandler
from posthog.dataclasses import frozen
from posthog.llm.gateway_client import team_distinct_id
@@ -93,60 +92,3 @@ def capture_embedding(*, team_id: int, input_tokens: int, trace_id: str, propert
)
except Exception:
logger.warning("business_knowledge.embedding_cost_capture_failed", team_id=team_id, exc_info=True)
-
-
-class _QuietCallbackHandler(CallbackHandler):
- """
- Records successful generations only. On a model error the base handler
- records the error message and calls ``capture_exception``. That message can
- contain prompt or output text, which privacy mode does not redact, and every
- caller here already handles the failure itself. The base handler also logs
- every prompt at DEBUG level, which privacy mode does not redact either.
- """
-
- def _log_debug_event(
- self,
- event_name: str,
- run_id: UUID,
- parent_run_id: UUID | None = None,
- **kwargs: Any,
- ) -> None:
- _ = event_name, run_id, parent_run_id, kwargs
-
- def on_llm_error(
- self,
- error: BaseException,
- *,
- run_id: UUID,
- parent_run_id: UUID | None = None,
- **kwargs: Any,
- ) -> None:
- _ = error, run_id, parent_run_id, kwargs
-
- def on_chain_error(
- self,
- error: BaseException,
- *,
- run_id: UUID,
- parent_run_id: UUID | None = None,
- **kwargs: Any,
- ) -> None:
- _ = error, run_id, parent_run_id, kwargs
-
-
-def trace_callback(team_id: int, *, trace_id: str, properties: Mapping[str, str | int]) -> CallbackHandler | None:
- """LangChain callback that captures ``$ai_generation`` for a model call made outside a PostHog AI graph."""
- # default_client stays None until the first module-level capture.
- client = posthoganalytics.default_client
- if client is None and not posthoganalytics.disabled:
- client = posthoganalytics.setup()
- if client is None:
- return None
- return _QuietCallbackHandler(
- client,
- distinct_id=team_distinct_id(team_id),
- trace_id=trace_id,
- # Prompts and outputs contain customer content, so only token counts reach analytics.
- privacy_mode=True,
- properties=dict(properties),
- )
diff --git a/products/business_knowledge/backend/logic.py b/products/business_knowledge/backend/logic.py
index 771848c59674..1a004ac93f0d 100644
--- a/products/business_knowledge/backend/logic.py
+++ b/products/business_knowledge/backend/logic.py
@@ -43,22 +43,18 @@ from django.db.models.functions import Cast, Coalesce, Substr
from django.utils import timezone
import structlog
-from langchain_core.messages import HumanMessage, SystemMessage
from posthog.api.embedding_worker import EmbeddingResponse, async_generate_embedding, generate_embedding
from posthog.dataclasses import frozen
from posthog.helpers.full_text_search import process_query
-from posthog.models.organization import OrganizationMembership
+from posthog.llm.gateway_client import build_ai_gateway_anthropic_client, team_distinct_id
from posthog.models.scoping import team_scope, with_team_scope
from posthog.models.scoping.manager import resolve_effective_team_id
from posthog.models.team.team import Team
-from posthog.models.user import User
from posthog.ph_client import feature_enabled_or_false
from posthog.security.url_validation import is_url_allowed
from posthog.sync import database_sync_to_async
-from ee.hogai.llm import MaxChatAnthropic
-
from . import crawl, discover, file_parse, html_parse, llm_telemetry, url_fetch
from .constants import (
BK_DRILLDOWN_DEFAULT_RADIUS,
@@ -71,6 +67,7 @@ from .constants import (
BK_QUERY_EMBEDDING_TIMEOUT,
BK_RERANK_MODEL,
BK_RERANK_SNIPPET_CHARS,
+ BK_RERANK_TIMEOUT,
BK_RRF_K,
BK_RRF_SCORE_FLOOR,
BK_SEARCH_MAX_LIMIT,
@@ -2935,18 +2932,6 @@ Return ONLY chunk_id UUIDs in order from most to least relevant, one UUID per li
Do not include any other text."""
-def _resolve_active_org_user(team: Team) -> User:
- membership = (
- OrganizationMembership.objects.select_related("user")
- .filter(organization=team.organization, user__is_active=True)
- .order_by("id")
- .first()
- )
- if not membership:
- raise RuntimeError(f"No active users in organization '{team.organization.name}' (team {team.id})")
- return membership.user
-
-
def _build_rerank_user_prompt(query: str, results: list[KnowledgeSearchResult]) -> str:
lines = [f"Query: {query}", "", "Candidates:"]
for index, result in enumerate(results, start=1):
@@ -3000,34 +2985,25 @@ def rerank_chunks(
valid_ids = {result.chunk_id for result in results}
id_to_result = {result.chunk_id: result for result in results}
- properties = llm_telemetry.retrieval_properties(
- team_id=team.id, trace=trace, feature=llm_telemetry.RETRIEVAL_RERANK_FEATURE
- )
try:
- user = _resolve_active_org_user(team)
- llm = MaxChatAnthropic(
- model=BK_RERANK_MODEL,
- streaming=False,
- user=user,
- team=team,
- max_tokens=1024,
- billable=False,
- inject_context=False,
- posthog_properties=properties,
- )
- # MaxChatAnthropic captures no $ai_generation by itself. The cost reaches analytics only through this callback.
- callback = llm_telemetry.trace_callback(team.id, trace_id=trace.trace_id, properties=properties)
- response = llm.invoke(
- [
- SystemMessage(content=_RERANK_SYSTEM_PROMPT),
- HumanMessage(content=_build_rerank_user_prompt(query, results)),
- ],
- config={"callbacks": [callback]} if callback is not None else None,
+ client = build_ai_gateway_anthropic_client(
+ ai_product=llm_telemetry.BK_AI_PRODUCT,
+ trace_id=trace.trace_id,
+ distinct_id=team_distinct_id(team.id),
+ team_id=team.id,
+ properties={"ai_feature": llm_telemetry.RETRIEVAL_RERANK_FEATURE, "surface": trace.surface},
)
- content = response.content
- if isinstance(content, list):
- content = "".join(str(item) for item in content)
- ranked_ids = _parse_reranked_chunk_ids(str(content), valid_ids)
+ with client.with_options(timeout=BK_RERANK_TIMEOUT, max_retries=3) as bounded:
+ response = bounded.messages.create(
+ model=BK_RERANK_MODEL,
+ max_tokens=1024,
+ system=_RERANK_SYSTEM_PROMPT,
+ messages=[{"role": "user", "content": _build_rerank_user_prompt(query, results)}],
+ # Prompts and outputs contain customer content, so only token counts reach analytics.
+ extra_headers={"X-PostHog-Billable": "false", "X-PostHog-Privacy-Mode": "true"},
+ )
+ content = "".join(block.text for block in response.content if block.type == "text")
+ ranked_ids = _parse_reranked_chunk_ids(content, valid_ids)
if ranked_ids is None:
return original_order[:top_k]
diff --git a/products/business_knowledge/backend/tests/test_rerank_chunks.py b/products/business_knowledge/backend/tests/test_rerank_chunks.py
index 8491b61d1b7f..92dd4b2db745 100644
--- a/products/business_knowledge/backend/tests/test_rerank_chunks.py
+++ b/products/business_knowledge/backend/tests/test_rerank_chunks.py
@@ -3,13 +3,36 @@ from uuid import UUID, uuid4
from posthog.test.base import BaseTest
from unittest.mock import MagicMock, patch
-from langchain_core.messages import AIMessage
+from anthropic.types import Message, TextBlock, Usage
from parameterized import parameterized
+from products.business_knowledge.backend.constants import BK_RERANK_TIMEOUT
from products.business_knowledge.backend.llm_telemetry import RetrievalTrace
from products.business_knowledge.backend.logic import KnowledgeSearchResult, rerank_chunks
+def _model_reply(text: str) -> Message:
+ return Message(
+ id="msg_test",
+ type="message",
+ role="assistant",
+ model="claude-test",
+ content=[TextBlock(type="text", text=text)],
+ stop_reason="end_turn",
+ stop_sequence=None,
+ usage=Usage(input_tokens=1, output_tokens=1),
+ )
+
+
+def _gateway_client(reply: Message) -> MagicMock:
+ client = MagicMock()
+ client.with_options.return_value.__enter__.return_value.messages.create.return_value = reply
+ return client
+
+
+_BUILD_CLIENT = "products.business_knowledge.backend.logic.build_ai_gateway_anthropic_client"
+
+
class TestRerankChunks(BaseTest):
def _make_result(self, content: str, *, chunk_id: UUID | None = None) -> KnowledgeSearchResult:
return KnowledgeSearchResult(
@@ -30,10 +53,9 @@ class TestRerankChunks(BaseTest):
def test_respects_top_k(self) -> None:
results = [self._make_result(f"content {index}") for index in range(5)]
reordered_ids = [results[2].chunk_id, results[0].chunk_id, results[4].chunk_id]
- mock_llm = MagicMock()
- mock_llm.invoke.return_value = AIMessage(content="\n".join(str(chunk_id) for chunk_id in reordered_ids))
+ client = _gateway_client(_model_reply("\n".join(str(chunk_id) for chunk_id in reordered_ids)))
- with patch("products.business_knowledge.backend.logic.MaxChatAnthropic", return_value=mock_llm):
+ with patch(_BUILD_CLIENT, return_value=client):
reranked = rerank_chunks(self.team, "query", results, top_k=2, trace=RetrievalTrace(surface="api"))
assert len(reranked) == 2
@@ -42,7 +64,7 @@ class TestRerankChunks(BaseTest):
@parameterized.expand(
[
("parse_failure", 3, "invalid_response"),
- ("model_failure", 2, "init_error"),
+ ("gateway_not_configured", 2, "init_error"),
]
)
def test_rrf_fallback_on_llm_failure(self, _name: str, num_results: int, failure_mode: str) -> None:
@@ -50,13 +72,11 @@ class TestRerankChunks(BaseTest):
top_k = 2
if failure_mode == "invalid_response":
- mock_llm = MagicMock()
- mock_llm.invoke.return_value = AIMessage(content="not valid uuids here")
- patch_kwargs: dict = {"return_value": mock_llm}
+ patch_kwargs: dict = {"return_value": _gateway_client(_model_reply("not valid uuids here"))}
else:
- patch_kwargs = {"side_effect": RuntimeError("llm unavailable")}
+ patch_kwargs = {"side_effect": ValueError("AI_GATEWAY_URL is not set")}
- with patch("products.business_knowledge.backend.logic.MaxChatAnthropic", **patch_kwargs):
+ with patch(_BUILD_CLIENT, **patch_kwargs):
reranked = rerank_chunks(self.team, "query", results, top_k=top_k, trace=RetrievalTrace(surface="api"))
assert [result.chunk_id for result in reranked] == [result.chunk_id for result in results[:top_k]]
@@ -66,26 +86,24 @@ class TestRerankChunks(BaseTest):
second = self._make_result("pricing tiers")
third = self._make_result("deployment guide")
results = [first, second, third]
- mock_llm = MagicMock()
- mock_llm.invoke.return_value = AIMessage(content=f"{third.chunk_id}\n{first.chunk_id}\n{second.chunk_id}")
+ client = _gateway_client(_model_reply(f"{third.chunk_id}\n{first.chunk_id}\n{second.chunk_id}"))
trace = RetrievalTrace(surface="support")
- with (
- patch("products.business_knowledge.backend.logic.MaxChatAnthropic", return_value=mock_llm),
- patch(
- "products.business_knowledge.backend.llm_telemetry.posthoganalytics",
- MagicMock(default_client=MagicMock(), disabled=False),
- ),
- ):
+ with patch(_BUILD_CLIENT, return_value=client) as build_client:
reranked = rerank_chunks(self.team, "how do refunds work", results, top_k=3, trace=trace)
assert [result.chunk_id for result in reranked] == [third.chunk_id, first.chunk_id, second.chunk_id]
- callback = mock_llm.invoke.call_args.kwargs["config"]["callbacks"][0]
- assert callback._trace_id == trace.trace_id
- assert callback._privacy_mode is True
- assert callback._properties == {
- "ai_product": "business_knowledge",
+ assert build_client.call_args.kwargs["trace_id"] == trace.trace_id
+ assert build_client.call_args.kwargs["ai_product"] == "business_knowledge"
+ assert build_client.call_args.kwargs["team_id"] == self.team.id
+ assert build_client.call_args.kwargs["properties"] == {
"ai_feature": "bk_retrieval_rerank",
- "team_id": self.team.id,
"surface": "support",
}
+ assert client.with_options.call_args.kwargs == {"timeout": BK_RERANK_TIMEOUT, "max_retries": 3}
+ assert client.with_options.return_value.__enter__.return_value.messages.create.call_args.kwargs[
+ "extra_headers"
+ ] == {
+ "X-PostHog-Billable": "false",
+ "X-PostHog-Privacy-Mode": "true",
+ }