:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
from datetime import UTC
from typing import Literal, cast
from django.utils import timezone
import structlog
from pydantic import BaseModel, Field
from posthog.schema import DateRange, LLMTrace, TracesQuery
from ee.hogai.context.insight.query_executor import AssistantQueryExecutor
from ee.hogai.tool import MaxTool
logger = structlog.get_logger(__name__)
SEARCH_TRACES_TOOL_PROMPT = """
Search for LLM traces based on criteria like date range, model, error status, and other properties.
# When to use this tool:
- User asks to find, search, or list LLM traces
- User asks about LLM usage, costs, latency, or errors
- User wants to filter traces by model, date, or other criteria
- User asks questions like "show me recent traces" or "what are the most expensive calls"
- User asks to "show more" or see "next page" of results (use the cursor from previous results)
# What this tool returns:
A formatted list of matching traces with their name, timestamp, latency, cost, token counts, and error count.
If more results are available, a cursor will be provided for pagination.
All timestamps are in UTC timezone.
# What this tool does NOT return:
This is metadata only. It does NOT include prompt/response content ($ai_input, $ai_output, $ai_output_choices, $ai_tools).
That content is retained for 30 days by default.
# Do not infer bugs or "no problems" from this list alone:
- To inspect what actually happened in a trace (the prompt, the model output, the tools called), use `read_data` with kind="llm_trace" and the trace's ID from this list.
- A trace older than the 30-day retention window still shows metadata here but has no content left to read — that is aged-out data, not evidence that nothing went wrong.
""".strip()
SEARCH_TRACES_QUERY_DESCRIPTION = """
User's question converted into a traces query.
IMPORTANT: When the user asks to "show more" or "next page", pass the cursor
from the previous response to continue pagination. Do NOT modify the query parameters.
# Query Structure
## dateRange (optional but recommended)
Time range for the query:
- date_from: Start date (relative like "-7d", "-30d" or absolute "2024-12-01")
- date_to: End date (null for "until now", or absolute date)
Default: last 7 days if not specified.
## properties (optional)
HogQL property filters for advanced filtering. Common properties:
- $ai_model: LLM model name (e.g., "gpt-4", "claude-3")
- $ai_is_error: Whether the trace had errors (true/false)
- $ai_span_name: Name of the trace/span
- $ai_provider: LLM provider name
## filterTestAccounts (optional)
- true: Exclude internal/test accounts
- false: Include all accounts (default)
## limit (optional)
Number of results to return (1-100, default 50). Use a higher limit to see more results
instead of paginating. Use filters and date ranges to narrow down results.
""".strip()
class SearchLLMTracesArgs(BaseModel):
query: TracesQuery = Field(description=SEARCH_TRACES_QUERY_DESCRIPTION)
cursor: str | None = Field(
default=None,
description="Pagination cursor from previous search results. Pass this to get the next page of results.",
)
class SearchLLMTracesTool(MaxTool):
name: Literal["search_llm_traces"] = "search_llm_traces"
args_schema: type[BaseModel] = SearchLLMTracesArgs
description: str = SEARCH_TRACES_TOOL_PROMPT
def get_required_resource_access(self):
return [("llm_analytics", "viewer")]
async def _arun_impl(self, query: TracesQuery, cursor: str | None = None) -> tuple[str, None]:
if query.limit is None or query.limit <= 0:
query.limit = 50
elif query.limit > 100:
query.limit = 100
if query.dateRange is None:
query.dateRange = DateRange(date_from="-7d")
if query.filterTestAccounts is None:
query.filterTestAccounts = False
# Pagination: the cursor is a stringified offset. This is simpler than timestamp-based
# cursors and works correctly because TracesQueryRunner's subquery and main query now
# use the same ordering (min(timestamp) DESC), so offset-based pagination is stable.
current_offset = 0
if cursor:
try:
current_offset = int(cursor)
except ValueError:
logger.warning("Invalid pagination cursor", cursor=cursor)
query.offset = current_offset
utc_now = timezone.now().astimezone(UTC)
executor = AssistantQueryExecutor(self._team, utc_now, user=self._user)
query_results = await executor.aexecute_query(query)
raw_results = cast(list[dict], query_results.get("results", []))
results = [LLMTrace.model_validate(r) for r in raw_results]
has_more = cast(bool, query_results.get("hasMore", False))
# TracesQueryRunner already trims the page to query.limit. This guard keeps the tool
# within its own contract if the runner ever returns the extra hasMore detection row.
if has_more and len(results) > query.limit:
results = results[: query.limit]
next_cursor = str(current_offset + len(results)) if has_more else None
content = self._format_results(results, has_more, next_cursor)
return content, None
def _format_results(self, results: list[LLMTrace], has_more: bool, next_cursor: str | None = None) -> str:
if not results:
return "No traces found matching your criteria. Try a wider date range or different filters."
total_count = len(results)
if total_count == 1:
content = "Found 1 trace:\n\n"
else:
content = f"Found {total_count} traces:\n\n"
for i, trace in enumerate(results, 1):
content += self._format_trace(i, trace)
content += (
"\nThis list is metadata only — prompt/response content ($ai_input, $ai_output, $ai_output_choices) "
'is not included. To inspect a trace\'s content, read it with read_data (kind="llm_trace") using the '
"trace's ID above (content is retained 30 days; older traces show metadata but no content)."
)
if has_more and next_cursor:
content += f'\nMore traces are available. Pass cursor="{next_cursor}" to see the next page.'
return content
def _format_trace(self, index: int, trace: LLMTrace) -> str:
name = trace.traceName or "Unnamed trace"
trace_id = trace.id
created_at = trace.createdAt
parts = [f"ID: {trace_id}"]
if trace.totalLatency is not None:
parts.append(f"Latency: {trace.totalLatency:.2f}s")
if trace.totalCost is not None:
parts.append(f"Cost: ${trace.totalCost:.4f}")
if trace.inputTokens is not None or trace.outputTokens is not None:
input_tokens = int(trace.inputTokens) if trace.inputTokens else 0
output_tokens = int(trace.outputTokens) if trace.outputTokens else 0
parts.append(f"Tokens: {input_tokens:,} in / {output_tokens:,} out")
if trace.errorCount is not None and trace.errorCount > 0:
parts.append(f"Errors: {int(trace.errorCount)}")
lines = [f"{index}. {name}"]
lines.append(f" {' | '.join(parts)}")
lines.append(f" Created: {created_at}")
lines.append("")
return "\n".join(lines)