:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
from typing import Any
import structlog
from posthog.schema import LLMTrace
from posthog.helpers.tiktoken_encoding import LLM_TOKEN_COUNT_PROXY_MODEL, get_tiktoken_encoding_for_model
from posthog.models.team.team import Team
logger = structlog.get_logger(__name__)
class LLMTracesSummarizerStringifier:
def __init__(self, team: Team):
self._team = team
self._token_encoder = get_tiktoken_encoding_for_model(LLM_TOKEN_COUNT_PROXY_MODEL)
self._stringified_trace_max_tokens = 5000
def stringify_traces(self, traces_chunk: list[LLMTrace]) -> dict[str, str]:
stringified_traces: dict[str, str] = {}
for trace in traces_chunk:
stringified_trace = self._stringify_trace_messages(trace)
if not stringified_trace:
continue
stringified_traces[trace.id] = stringified_trace
return stringified_traces
def _stringify_trace_messages(self, trace: LLMTrace) -> str | None:
stringified_messages: list[str] = []
# TODO: Iterate full conversations (traces combined) instead of just traces, as it leads to duplicates
messages = trace.outputState.get("messages") if trace.outputState else []
for message in messages:
stringified_message = self.stringify_message(message=message, trace_id=trace.id)
# Skip empty messages
if not stringified_message:
continue
# Check that the previous message isn't identical
if stringified_messages and stringified_messages[-1] == stringified_message:
continue
stringified_messages.append(stringified_message)
# If no messages, skip the trace
if not stringified_messages:
return None
# If human didn't respond to any AI messages (no interaction), skip the trace
no_interaction_found = True
for i, message in enumerate(stringified_messages):
if message.startswith("human") and i > 0 and stringified_messages[i - 1].startswith("ai"):
no_interaction_found = False
break
if no_interaction_found:
return None
# Combine into string
stringified_messages_str = "\n\n".join(stringified_messages)
# Check if the trace is too long for summarization
num_tokens = len(self._token_encoder.encode(stringified_messages_str))
if num_tokens > self._stringified_trace_max_tokens:
logger.warning(
f"Trace {trace.id} from team {self._team.id} stringified version is too long ({num_tokens} tokens > {self._stringified_trace_max_tokens})"
"for summarization when summarizing LLM traces, skipping"
)
return None
return stringified_messages_str
@staticmethod
def _stringify_answer(message: dict[str, Any]) -> str | None:
answer_kind = message["answer"]["kind"]
message_content = f"*AI displayed a {answer_kind}*"
if not message_content:
return None
return f"ai/answer: {message_content}"
@staticmethod
def _stringify_ai_message(message: dict[str, Any]) -> str | None:
message_content = message["content"]
tools_called = []
for tc in message.get("tool_calls") or []:
if tc.get("type") != "tool_call":
continue
tools_called.append(tc.get("name"))
if tools_called:
tool_content = f"*AI called tools: {', '.join(tools_called)}*"
message_content += f" {tool_content}" if message_content else tool_content
if not message_content:
return None
return f"ai: {message_content}"
@staticmethod
def _stringify_human_message(message: dict[str, Any]) -> str | None:
message_content = message["content"]
if not message_content:
return None
return f"human: {message_content}"
def stringify_message(self, message: dict[str, Any], trace_id: str) -> str | None:
message_type = message.get("type")
if not message_type:
logger.warning(
f"Message {message.get('id')} for trace {trace_id} from team {self._team.id} has no type, skipping"
)
return None
try:
# Answers
if message.get("answer"):
return self._stringify_answer(message)
# Messages
if message_type == "ai":
return self._stringify_ai_message(message)
if message_type == "human":
return self._stringify_human_message(message)
if message_type == "context": # Skip context messages
return None
if message_type == "tool":
return None
# Ignore other message types
# TODO: Decide if there's a need for other message types
return None
except Exception as err:
logger.exception(
f"Error stringifying message {message_type} ({err}) for trace {trace_id} from team {self._team.id}:\n{message}",
error=str(err),
)
return None