:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import re
import json
# nosemgrep: python.lang.security.use-defused-xml.use-defused-xml (XML generation only, no parsing - no XXE risk)
import xml.etree.ElementTree as ET
from collections.abc import Mapping, Sequence
from datetime import datetime
from typing import Any, Optional, TypeVar, Union, cast
from urllib.parse import urlparse
from uuid import uuid4
from jsonref import replace_refs
from langchain_core.messages import (
AIMessage,
AIMessageChunk,
BaseMessage,
HumanMessage as LangchainHumanMessage,
merge_message_runs,
)
from posthog.schema import (
AssistantFunnelsQuery,
AssistantHogQLQuery,
AssistantMessage,
AssistantMessageMetadata,
AssistantRetentionQuery,
AssistantToolCall,
AssistantToolCallMessage,
AssistantTrendsQuery,
CachedTeamTaxonomyQueryResponse,
ContextMessage,
FunnelsQuery,
HogQLQuery,
HumanMessage,
MaxEventContext,
RetentionQuery,
TeamTaxonomyQuery,
TrendsQuery,
)
from posthog.dataclasses import frozen
from posthog.event_usage import EventSource
from posthog.hogql_queries.ai.team_taxonomy_query_runner import LOOKBACK_DAYS, TeamTaxonomyQueryRunner
from posthog.hogql_queries.query_runner import ExecutionMode
from posthog.models import Team, User
from posthog.security.llm_prompt_sanitization import sanitize_user_text
from posthog.settings import EE_AVAILABLE
from posthog.taxonomy.taxonomy import CORE_FILTER_DEFINITIONS_BY_GROUP, is_hidden_from_assistant
from products.event_definitions.backend.models.property_definition import effective_project_id_expr
from ee.hogai.utils.anthropic import SUPPORTED_ANTHROPIC_BLOCKS
from ee.hogai.utils.types.base import (
ArtifactRefMessage,
AssistantDispatcherEvent,
AssistantMessageUnion,
ConversationTitleAction,
)
def sanitize_for_system_reminder(text: str) -> str:
"""Neutralize system_reminder tags to prevent framing spoofs in user-provided content."""
return re.sub(r"<(\s*/?\s*system_reminder\b[^>]*)>", r"<\1>", text, flags=re.IGNORECASE)
BK_DRILLDOWN_HANDLE_RE = re.compile(r"`?\[bk-doc=[^\]]*\]`?")
def strip_bk_drilldown_handles(text: str) -> str:
"""Remove BK drill-down handles from text. No-op when absent."""
result = BK_DRILLDOWN_HANDLE_RE.sub("", text)
return re.sub(r" +", " ", result)
def remove_line_breaks(line: str) -> str:
return line.replace("\n", " ")
_CONTROL_CHARS_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")
# Descriptions are an unbounded TextField; cap each one so a single oversized description can't
# blow up every team member's prompt. The taxonomy already bounds the number of events per prompt.
MAX_EVENT_DESCRIPTION_LENGTH = 500
# Matches CAPTURE_V1_MAX_EVENT_NAME_LENGTH in rust/capture, which rejects a longer name at ingestion.
MAX_EVENT_NAME_LENGTH = 200
# Covers the User model's limits: 150 characters each for first and last name, and 254 for an email.
MAX_EVENT_OWNER_FIELD_LENGTH = 301
NOT_SEEN_RECENTLY_MARKER = f"(not seen in the last {LOOKBACK_DAYS} days)"
NOT_SEEN_RECENTLY_LEGEND = (
f"Events marked {NOT_SEEN_RECENTLY_MARKER} are listed for reference only. This project has sent none of them "
"recently, so never present them as data it is collecting."
)
@frozen
class EventTaxonomySnapshot:
"""One read of a team's event taxonomy, with the freshness facts a caller needs to trust it."""
events: list[dict[str, Any]]
has_more: bool
computed_at: datetime
def _taxonomy_snapshot_legend(computed_at: datetime) -> str:
"""State what the counts measure and when they were measured.
Other surfaces read the same project over their own windows, and PostHog system events that are
not useful for analysis never reach this list at all. Without the window, the snapshot time and
the omission on the response, a caller that compares two surfaces reads an honest difference as
a broken taxonomy.
"""
return (
f"Taxonomy snapshot taken at {computed_at.isoformat()}. Counts cover the last {LOOKBACK_DAYS} days, so a "
"surface that reads a shorter or longer window can disagree with this list. PostHog system events that are "
"not useful for analysis are left out, so a surface that reads raw event traffic can show an event that is "
"absent here."
)
def sanitize_event_description(text: str) -> str:
"""Neutralize an untrusted event description before it goes into the model's context.
Event definition descriptions (and per-conversation context descriptions) are user-controlled
project metadata, so they're treated as untrusted data: an editor could embed instructions that
reach another user's agent session verbatim. Collapse control characters and whitespace so the
text can't break out of its line, cap the length, and neutralize system_reminder framing.
"""
collapsed = re.sub(r"\s+", " ", _CONTROL_CHARS_RE.sub(" ", text)).strip()
if len(collapsed) > MAX_EVENT_DESCRIPTION_LENGTH:
collapsed = collapsed[:MAX_EVENT_DESCRIPTION_LENGTH].rstrip() + "…"
return sanitize_for_system_reminder(collapsed)
def filter_and_merge_messages(
messages: Sequence[AssistantMessageUnion],
entity_filter: Union[tuple[type[AssistantMessageUnion], ...], type[AssistantMessageUnion]] = (
AssistantMessage,
ArtifactRefMessage,
),
) -> list[AssistantMessageUnion]:
"""
Filters and merges the message history to be consumable by agents. Returns human and AI messages.
"""
stack: list[LangchainHumanMessage] = []
filtered_messages: list[AssistantMessageUnion] = []
def _merge_stack(stack: list[LangchainHumanMessage]) -> list[HumanMessage]:
return [
HumanMessage(content=langchain_message.content, id=langchain_message.id)
for langchain_message in merge_message_runs(stack)
]
for message in messages:
if isinstance(message, HumanMessage):
stack.append(LangchainHumanMessage(content=message.content, id=message.id))
elif isinstance(message, entity_filter):
if stack:
filtered_messages += _merge_stack(stack)
stack = []
filtered_messages.append(message)
if stack:
filtered_messages += _merge_stack(stack)
return filtered_messages
T = TypeVar("T", bound=AssistantMessageUnion)
def find_last_message_of_type(messages: Sequence[AssistantMessageUnion], message_type: type[T]) -> Optional[T]:
return next((msg for msg in reversed(messages) if isinstance(msg, message_type)), None)
def slice_messages_to_conversation_start(
messages: Sequence[AssistantMessageUnion], start_id: Optional[str] = None
) -> Sequence[AssistantMessageUnion]:
result = []
for msg in messages:
result.append(msg)
if msg.id == start_id:
break
return result
def dereference_schema(schema: dict) -> dict:
new_schema: dict = replace_refs(schema, proxies=False, lazy_load=False)
if "$defs" in new_schema:
new_schema.pop("$defs")
return new_schema
def find_start_message_idx(messages: Sequence[AssistantMessageUnion], start_id: str | None = None) -> int:
for idx in range(len(messages) - 1, -1, -1):
msg = messages[idx]
if isinstance(msg, HumanMessage) and msg.id == start_id:
return idx
return 0
def find_start_message(messages: Sequence[AssistantMessageUnion], start_id: str | None = None) -> HumanMessage | None:
if not messages:
return None
index = find_start_message_idx(messages, start_id)
return cast(HumanMessage, messages[index])
def should_output_assistant_message(candidate_message: Any) -> bool:
"""
This is used to filter out messages that are not useful for the user.
Filter out empty assistant messages and context messages.
"""
if isinstance(candidate_message, AssistantMessage):
if (
(candidate_message.tool_calls is None or len(candidate_message.tool_calls) == 0)
and len(candidate_message.content) == 0
and candidate_message.meta is None
):
# Empty assistant message
return False
# Filter out context messages
if isinstance(candidate_message, ContextMessage):
return False
return True
def convert_tool_messages_to_dict(messages: Sequence[AssistantMessageUnion]) -> Mapping[str, AssistantToolCallMessage]:
"""Converts `AssistantToolCallMessage` messages to a dictionary mapping tool call id to message."""
return {message.tool_call_id: message for message in messages if isinstance(message, AssistantToolCallMessage)}
def _process_events_data(
events_in_context: list[MaxEventContext],
team: Team,
user: User,
limit: int | None = None,
offset: int | None = None,
event_source: EventSource = EventSource.POSTHOG_AI,
) -> EventTaxonomySnapshot:
"""Common logic for processing events and building event data."""
query = TeamTaxonomyQuery(limit=limit, offset=offset)
response = TeamTaxonomyQueryRunner(query, team, user=user).run(
ExecutionMode.RECENT_CACHE_CALCULATE_ASYNC_IF_STALE_AND_BLOCKING_ON_MISS,
user=user,
analytics_props={"source": event_source},
)
if not isinstance(response, CachedTeamTaxonomyQueryResponse):
raise ValueError("Failed to generate events prompt.")
# The runner pads a complete response with well-known event names at count 0, so a zero count
# means the project has no such event in the query window, not that the event is merely rare.
not_seen_recently = {item.event for item in response.results if item.count == 0}
events: list[str] = [
# Add "All events" to the mapping
"All events",
]
for item in response.results:
event_def = CORE_FILTER_DEFINITIONS_BY_GROUP.get("events", {}).get(item.event)
if event_def and is_hidden_from_assistant(event_def):
continue # Skip system or ignored events (safety net, already filtered in SQL)
events.append(item.event)
event_to_description: dict[str, str] = {}
for event in events_in_context:
if event.name and event.name not in events:
events.append(event.name)
if event.name and event.description:
event_to_description[event.name] = event.description
# Create a set of event names from context for efficient lookup
context_event_names = {event.name for event in events_in_context if event.name}
# Fall back to the user-authored descriptions stored on event definitions for any custom
# events that aren't covered by the core taxonomy or the conversation context.
db_event_descriptions = _get_event_definition_descriptions(team, events, event_to_description)
processed_events = []
for event_name in events:
# A project's write token is public, so a captured name is untrusted: left raw, one holding
# a line break writes its own `#` legend line into the listing `format_events_yaml` builds.
safe_name = sanitize_user_text(event_name, MAX_EVENT_NAME_LENGTH)
if not safe_name:
continue
event_data: dict[str, Any] = {"name": safe_name}
if event_name in not_seen_recently:
event_data["not_seen_recently"] = True
if event_core_definition := CORE_FILTER_DEFINITIONS_BY_GROUP["events"].get(event_name):
# Only skip if it's not in context (context events should always be included)
if event_name not in context_event_names and is_hidden_from_assistant(event_core_definition):
continue # Skip irrelevant events but keep events the user has added to the context
if core_description := _format_core_event_description(event_core_definition):
event_data["description"] = core_description
elif event_name in event_to_description:
event_data["description"] = sanitize_event_description(event_to_description[event_name])
elif event_name in db_event_descriptions:
event_data["description"] = sanitize_event_description(db_event_descriptions[event_name])
processed_events.append(event_data)
return EventTaxonomySnapshot(
events=processed_events,
has_more=bool(response.hasMore),
computed_at=response.last_refresh,
)
def _format_core_event_description(event_core_definition: Mapping[str, Any]) -> str | None:
"""Build a single-line description for a core taxonomy event, prefixed with its label."""
description = event_core_definition.get("description")
if not description:
return None
if label := event_core_definition.get("label_llm") or event_core_definition.get("label"):
description = f"{label}. {description}"
return remove_line_breaks(description)
def get_event_description(team: Team, event_name: str) -> str | None:
"""Return a human-readable description for a single event, or None if none is known.
Mirrors the precedence used when listing events: the built-in core taxonomy description wins,
otherwise fall back to the user-authored description stored on the event definition (EE only).
Used to surface a description on the single-event properties path so the agent doesn't have to
list every event to learn what one event means.
"""
if event_core_definition := CORE_FILTER_DEFINITIONS_BY_GROUP["events"].get(event_name):
return _format_core_event_description(event_core_definition)
if description := _get_event_definition_descriptions(team, [event_name], {}).get(event_name):
return sanitize_event_description(description)
return None
def get_event_owner(team: Team, event_name: str) -> str | None:
"""Return the owner of a single event as "Name (email)", or None if the event has no owner.
Only the enterprise `EventDefinition` model carries an `owner`, so this is a no-op on non-EE builds.
Core taxonomy events are looked up too: an owner only ever comes from the event definition.
"""
if not EE_AVAILABLE:
return None
from ee.models.event_definition import (
EnterpriseEventDefinition, # noqa: PLC0415 — EE-only model, keep off the OSS import path
)
# Event definitions belong to the project, so an environment sees the owner set from any other one.
owner = (
EnterpriseEventDefinition.objects.alias(effective_project_id=effective_project_id_expr())
.filter(effective_project_id=team.project_id, name=event_name, owner__isnull=False)
.values_list("owner__first_name", "owner__last_name", "owner__email")
.first()
)
if owner is None:
return None
first_name, last_name, email = owner
# Users write their own names, so a name is untrusted text like an event description.
name = sanitize_user_text(f"{first_name} {last_name}", MAX_EVENT_OWNER_FIELD_LENGTH)
email = sanitize_user_text(email, MAX_EVENT_OWNER_FIELD_LENGTH)
return f"{name} ({email})" if name else email
def _get_event_definition_descriptions(
team: Team, event_names: list[str], already_described: dict[str, str]
) -> dict[str, str]:
"""Map event name -> user-authored description from the team's event definitions.
Only the enterprise `EventDefinition` model carries a `description` field, so this is a no-op
on non-EE builds. Skips core taxonomy events and events already described via context, and
runs a single batched query rather than one per event.
"""
if not EE_AVAILABLE:
return {}
from ee.models.event_definition import (
EnterpriseEventDefinition, # noqa: PLC0415 — EE-only model, keep off the OSS import path
)
names_to_fetch = [
name
for name in event_names
if name not in already_described
and name != "All events"
and name not in CORE_FILTER_DEFINITIONS_BY_GROUP["events"]
]
if not names_to_fetch:
return {}
rows = (
EnterpriseEventDefinition.objects.filter(team_id=team.pk, name__in=names_to_fetch)
.exclude(description__isnull=True)
.exclude(description="")
.values_list("name", "description")
)
return {name: description for name, description in rows if description}
def format_events_xml(events_in_context: list[MaxEventContext], team: Team, user: User) -> str:
snapshot = _process_events_data(events_in_context, team, user)
root = ET.Element("defined_events")
for event_data in snapshot.events:
event_tag = ET.SubElement(root, "event")
name_tag = ET.SubElement(event_tag, "name")
name_tag.text = event_data["name"]
if "description" in event_data:
desc_tag = ET.SubElement(event_tag, "description")
desc_tag.text = event_data["description"]
if event_data.get("not_seen_recently"):
ET.SubElement(event_tag, "not_seen_recently").text = "true"
return ET.tostring(root, encoding="unicode")
def format_events_yaml(
events_in_context: list[MaxEventContext],
team: Team,
user: User,
limit: int | None = None,
offset: int | None = None,
event_source: EventSource = EventSource.POSTHOG_AI,
) -> str:
snapshot = _process_events_data(
events_in_context, team, user, limit=limit, offset=offset, event_source=event_source
)
formatted_events = ["events:"]
any_not_seen_recently = False
for event_data in snapshot.events:
name = event_data["name"]
description = event_data.get("description", "")
line = f"- `{name}` - {description}" if description else f"- `{name}`"
if event_data.get("not_seen_recently"):
any_not_seen_recently = True
line += f" {NOT_SEEN_RECENTLY_MARKER}"
formatted_events.append(line)
if any_not_seen_recently:
formatted_events.append(f"\n# {NOT_SEEN_RECENTLY_LEGEND}")
if snapshot.has_more:
next_offset = (offset or 0) + (limit or 500)
formatted_events.append(f"\n# More events available. To fetch the next page, use offset={next_offset}")
formatted_events.append(f"\n# {_taxonomy_snapshot_legend(snapshot.computed_at)}")
return "\n".join(formatted_events)
def extract_content_from_ai_message(response: BaseMessage) -> str:
"""
Extracts the content from a BaseMessage, supporting both reasoning and non-reasoning responses.
"""
if isinstance(response.content, list):
text_parts: list[str] = []
for content_item in response.content:
if isinstance(content_item, dict) and "type" in content_item and content_item["type"] == "text":
text_parts.append(content_item["text"])
elif isinstance(content_item, str):
text_parts.append(content_item)
return "".join(text_parts)
return str(response.content)
def extract_thinking_from_ai_message(response: BaseMessage) -> list[dict[str, Any]]:
thinking: list[dict[str, Any]] = []
for content in response.content:
# Anthropic style reasoning
if isinstance(content, dict) and "type" in content:
if content["type"] in SUPPORTED_ANTHROPIC_BLOCKS:
thinking.append(content)
if response.additional_kwargs.get("reasoning") and (
summary := response.additional_kwargs["reasoning"].get("summary")
):
# OpenAI style reasoning
thinking.append(
{
"type": "thinking",
"thinking": summary[0]["text"],
}
)
return thinking
def normalize_ai_message(message: AIMessage | AIMessageChunk) -> list[AssistantMessage]:
_create_blank_assistant_message = lambda: AssistantMessage(
content="", id=None if isinstance(message, AIMessageChunk) else str(uuid4()), tool_calls=[]
)
if isinstance(message.content, list):
messages: list[AssistantMessage] = [_create_blank_assistant_message()]
for content_item in message.content:
if isinstance(content_item, dict) and "type" in content_item:
if content_item["type"] == "server_tool_use":
# Server tool use requires starting a new AssistantMessage for correct presentation of the subsequent output
messages.append(_create_blank_assistant_message())
# Also we need to parse `input` from `partial_json`, as weirdly server tool uses in LangChain
# have an empty `input` even `partial_json` is ready and clearly has args
if content_item.get("partial_json"):
try:
content_item["input"] = json.loads(content_item["partial_json"])
except json.JSONDecodeError:
pass
if content_item["type"] == "text":
if "text" in content_item:
messages[-1].content += content_item["text"]
if "citations" in content_item:
messages[-1].content += "".join(
f" [({urlparse(citation['url']).netloc})]({citation['url']})" # Must have space in front
for citation in content_item["citations"]
)
elif content_item["type"] in SUPPORTED_ANTHROPIC_BLOCKS:
# All of these blocks must be preserved in their original order for Anthropic interleaved thinking
if not messages[-1].meta:
messages[-1].meta = AssistantMessageMetadata(thinking=[])
messages[-1].meta.thinking.append(content_item) # type: ignore
elif isinstance(content_item, str):
messages[-1].content += content_item
else:
content = extract_content_from_ai_message(message)
thinking = extract_thinking_from_ai_message(message)
messages = [
AssistantMessage(
id=None if isinstance(message, AIMessageChunk) else str(uuid4()),
content=content,
meta=AssistantMessageMetadata(thinking=thinking) if thinking else None,
)
]
# Regular tool calls are added separately to the last message, as their args must be fully complete to be JSON-valid
if isinstance(message, AIMessageChunk):
tool_calls = [
AssistantToolCall(
id=tool_call["id"],
name=tool_call["name"],
args=(tool_call["args"] if isinstance(tool_call["args"], dict) else {}),
)
for tool_call in message.tool_call_chunks
if tool_call["id"] is not None and tool_call["name"] is not None
]
else:
tool_calls = [
AssistantToolCall(id=tool_call["id"], name=tool_call["name"], args=tool_call["args"] or {})
for tool_call in message.tool_calls
]
messages[-1].tool_calls = tool_calls
if isinstance(message, AIMessageChunk):
for i, final_message in enumerate(messages):
final_message.id = f"temp-{i}" # Assign each ephemeral message an index-based temp ID
for msg in messages:
if msg.content:
msg.content = strip_bk_drilldown_handles(msg.content)
return messages
def cast_assistant_query(
query: AssistantTrendsQuery | AssistantFunnelsQuery | AssistantRetentionQuery | AssistantHogQLQuery,
) -> TrendsQuery | FunnelsQuery | RetentionQuery | HogQLQuery:
"""
Convert AssistantQuery types to regular Query types that the frontend expects.
"""
if query.kind == "TrendsQuery":
return TrendsQuery(**query.model_dump())
elif query.kind == "FunnelsQuery":
return FunnelsQuery(**query.model_dump())
elif query.kind == "RetentionQuery":
return RetentionQuery(**query.model_dump())
elif query.kind == "HogQLQuery":
return HogQLQuery(**query.model_dump())
else:
raise ValueError(f"Unsupported query type: {query.kind}")
def build_insight_url(id: str) -> str:
"""Build the URL for an insight.
Unprefixed by `/project/<id>`: these URLs are handed to the model, which is instructed to omit
that prefix, and the app resolves them against the project the user is already in.
"""
return f"/insights/{id}"
def build_dashboard_url(id: int) -> str:
"""Build the URL for a dashboard. Unprefixed, for the same reason as `build_insight_url`."""
return f"/dashboard/{id}"
def extract_stream_update(update: Any) -> Any:
# Handle old LangGraph tuple format
if update[1] == "custom":
# Custom streams come from a tool call
# If it's a LangGraph-based chunk, we remove the first two elements, which are "custom" and the parent graph namespace
update = update[2]
if isinstance(update, (AssistantDispatcherEvent, ConversationTitleAction)):
return update
update = update[1:] # we remove the first element, which is the node/subgraph node name
return update
def insert_messages_before_start(
messages: Sequence[AssistantMessageUnion],
new_messages: Sequence[AssistantMessageUnion],
start_id: str | None = None,
) -> list[AssistantMessageUnion]:
# Insert context messages right before the start message
start_idx = find_start_message_idx(messages, start_id)
return [*messages[:start_idx], *new_messages, *messages[start_idx:]]
def sort_schema_properties(
schema: dict, top_level_order: list[str], property_order_map: dict[str, list[str]] | None = None
) -> dict:
"""
Sorts properties in the JSON schema according to predefined order.
Uses startswith matching for schema names to handle variants.
Modifies the schema in place.
"""
default_order_map = {
# Events and Actions nodes - prioritize kind and identifying field
"AssistantTrendsEventsNode": ["kind", "event"],
"AssistantTrendsActionsNode": ["kind", "id"],
# Generic property filters - type first for all variants
"AssistantGenericPropertyFilter": ["type", "key", "operator", "value"],
# Group property filters - type first for all variants
"AssistantGroupPropertyFilter": ["type", "key", "operator", "value", "group_type_index"],
}
full_order_map = {**default_order_map, **(property_order_map or {})}
def get_property_order(schema_name: str) -> list[str]:
"""Get property order for a schema name using startswith matching."""
for prefix, order in full_order_map.items():
if schema_name.startswith(prefix):
return order
return []
def reorder_properties(obj: dict, order: list[str]) -> dict:
"""Reorders properties dict according to the order list."""
if "properties" not in obj:
return obj
properties = obj["properties"]
if not order:
return obj
# Create new ordered dict
ordered_properties = {}
# Add properties in specified order
for key in order:
if key in properties:
ordered_properties[key] = properties[key]
# Add remaining properties not in the order list
for key, value in properties.items():
if key not in ordered_properties:
ordered_properties[key] = value
obj["properties"] = ordered_properties
return obj
# Reorder top-level properties using provided order
if "properties" in schema:
schema = reorder_properties(schema, top_level_order)
# Reorder properties in $defs
if "$defs" in schema:
defs = schema["$defs"]
for def_name in defs:
order = get_property_order(def_name)
if order:
defs[def_name] = reorder_properties(defs[def_name], order)
return schema