Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

17.5 KB
Raw Blame
import json
from abc import ABC, abstractmethod
from collections.abc import Callable, Sequence
from typing import Any, TypeVar, cast
from uuid import uuid4

from langchain_anthropic import ChatAnthropic
from langchain_core.language_models import BaseChatModel
from langchain_core.messages import (
    AIMessage as LangchainAIMessage,
    BaseMessage,
    HumanMessage as LangchainHumanMessage,
)
from langchain_core.tools import BaseTool
from langchain_core.utils.function_calling import convert_to_openai_tool
from pydantic import BaseModel, ValidationError

from posthog.schema import (
    AgentMode,
    AssistantMessage,
    AssistantTool,
    AssistantToolCallMessage,
    ContextMessage,
    HumanMessage,
)

from posthog.sync import database_sync_to_async

from ee.hogai.context.prompts import CONTEXT_INITIAL_MODE_PROMPT
from ee.hogai.core.agent_modes.prompts import ROOT_AGENT_MODE_REMINDER_PROMPT, ROOT_TODO_REMINDER_PROMPT
from ee.hogai.tools.todo_write import TodoWriteTool
from ee.hogai.utils.helpers import find_start_message
from ee.hogai.utils.prompt import format_prompt_string
from ee.hogai.utils.types import AssistantMessageUnion

T = TypeVar("T", bound=AssistantMessageUnion)

LangchainTools = Sequence[dict[str, Any] | type | Callable | BaseTool]


class InsertionResult(BaseModel):
    messages: Sequence[AssistantMessageUnion]
    updated_start_id: str | None
    updated_window_start_id: str


class ConversationCompactionManager(ABC):
    """
    Manages conversation window boundaries, message filtering, and summarization decisions.
    """

    CONVERSATION_WINDOW_SIZE = 400_000
    """
    Determines the maximum number of tokens allowed in the conversation window.

    Compaction blocks the user's turn for the length of a full summarization call, so this is set
    well below the 1M context both the root model and the summarizer support: the cost of carrying
    a larger window on every turn is mostly cache reads, while the cost of compacting is dead time
    the user watches. Raising this trades per-turn input tokens for far fewer compactions.
    """
    APPROXIMATE_TOKEN_LENGTH = 4
    """
    Determines the approximate number of characters per token.
    """

    def find_window_boundary(self, messages: Sequence[T], max_messages: int = 10, max_tokens: int = 1000) -> str | None:
        """
        Find the optimal window start ID based on message count and token limits.
        Ensures the window starts at a human or assistant message.
        """

        new_window_id: str | None = None
        for message in reversed(messages):
            # Handle limits before assigning the window ID.
            max_tokens -= self._get_estimated_assistant_message_tokens(message)
            max_messages -= 1
            if max_tokens < 0 or max_messages < 0:
                break

            # Assign the new new window ID.
            if message.id is not None:
                if isinstance(message, HumanMessage):
                    new_window_id = message.id
                if isinstance(message, AssistantMessage):
                    new_window_id = message.id

        return new_window_id

    def get_messages_in_window(self, messages: Sequence[T], window_start_id: str | None = None) -> Sequence[T]:
        """
        Filter messages to only those within the conversation window.
        """
        if window_start_id is not None:
            return self._get_conversation_window(messages, window_start_id)
        return messages

    async def should_compact_conversation(
        self, model: BaseChatModel, messages: list[BaseMessage], tools: LangchainTools | None = None, **kwargs
    ) -> bool:
        """
        Determine if the conversation should be summarized based on token count.
        Avoids summarizing if there are only two human messages or fewer.
        """
        return await self.calculate_token_count(model, messages, tools, **kwargs) > self.CONVERSATION_WINDOW_SIZE

    async def calculate_token_count(
        self, model: BaseChatModel, messages: list[BaseMessage], tools: LangchainTools | None = None, **kwargs
    ) -> int:
        """
        Calculate the token count for a conversation.
        """
        # Avoid summarizing the conversation if there is only two human messages.
        human_messages = [message for message in messages if isinstance(message, LangchainHumanMessage)]
        if tools:
            # Filter out server-side tools for token counting purposes
            tools = [
                tool
                for tool in tools
                if not (isinstance(tool, dict) and tool.get("type", "").startswith("web_search_"))
            ]
        if len(human_messages) <= 2:
            tool_tokens = self._get_estimated_tools_tokens(tools) if tools else 0
            return sum(self._get_estimated_langchain_message_tokens(message) for message in messages) + tool_tokens
        return await self._get_token_count(model, messages, tools, **kwargs)

    def update_window(
        self,
        messages: Sequence[T],
        summary_message: ContextMessage,
        agent_mode: AgentMode,
        start_id: str | None = None,
        *,
        window_start_id: str | None = None,
    ) -> InsertionResult:
        """Finds the optimal position to insert the summary message in the conversation window."""
        active_window = self.get_messages_in_window(messages, window_start_id)
        boundary_id = self.find_window_boundary(active_window, max_messages=16, max_tokens=2048)
        start_message = find_start_message(messages, start_id)
        if not isinstance(start_message, HumanMessage):
            raise ValueError("Start message not found")
        if summary_message.id is None:
            raise ValueError("Summary message must have an ID")

        boundary_idx = next(
            (idx for idx, message in enumerate(messages) if boundary_id is not None and message.id == boundary_id),
            len(messages),
        )
        retained_messages = messages[boundary_idx:]
        inserted_messages: list[AssistantMessageUnion] = [summary_message]
        if not any(message is start_message for message in retained_messages):
            # The original request stays in the archived history, and the thread renders every HumanMessage,
            # so the copy the model needs is a ContextMessage: it reaches the model as a user turn but never
            # reaches the rendered thread or the stream.
            inserted_messages.append(ContextMessage(content=start_message.content, id=str(uuid4())))

        # Keep the display history intact; only the model's window moves past archived messages.
        updated_messages: Sequence[AssistantMessageUnion] = [
            *messages[:boundary_idx],
            *inserted_messages,
            *retained_messages,
        ]
        updated_messages = self._insert_reminders_after_summary(
            updated_messages,
            summary_message.id,
            agent_mode,
            all_messages=messages,
            window_messages=retained_messages,
        )

        return InsertionResult(
            messages=updated_messages,
            updated_start_id=start_message.id,
            updated_window_start_id=summary_message.id,
        )

    def _insert_reminders_after_summary(
        self,
        messages: Sequence[T],
        summary_id: str,
        agent_mode: AgentMode,
        all_messages: Sequence[T] | None = None,
        window_messages: Sequence[T] | None = None,
    ) -> Sequence[T]:
        """
        Insert both todo reminder (if needed) and mode reminder (if needed) after summary.
        Order: summary → todo reminder → mode reminder → rest

        Args:
            messages: The messages list to insert into
            summary_id: ID of the summary message
            agent_mode: Current agent mode for mode reminder
            all_messages: Full message history for finding todo (defaults to messages if None)
            window_messages: Messages in the new window for checking presence (defaults to messages if None)

        Returns:
            Updated messages with reminders inserted
        """
        if all_messages is None:
            all_messages = messages
        if window_messages is None:
            window_messages = messages

        # Determine what needs to be inserted
        reminders_to_insert: list[T] = []

        # 1. Todo reminder (if needed)
        if todo_reminder := self._get_todo_reminder_message(all_messages, window_messages):
            reminders_to_insert.append(cast(T, todo_reminder))

        # 2. Mode reminder (if needed)
        if mode_reminder := self._get_mode_message_with_context(window_messages, all_messages, agent_mode):
            reminders_to_insert.append(cast(T, mode_reminder))

        # If nothing to insert, return original messages
        if not reminders_to_insert:
            return messages

        # Insert all reminders right after summary
        summary_idx = next(i for i, msg in enumerate(messages) if msg.id == summary_id)
        result: Sequence[T] = [
            *messages[: summary_idx + 1],
            *reminders_to_insert,
            *messages[summary_idx + 1 :],
        ]
        return result

    def _get_estimated_assistant_message_tokens(self, message: AssistantMessageUnion) -> int:
        """
        Estimate token count for a message using character/4 heuristic.
        """
        char_count = 0
        if isinstance(message, HumanMessage | ContextMessage):
            char_count = len(message.content)
        elif isinstance(message, AssistantMessage):
            char_count = len(message.content) + sum(
                len(json.dumps(m.args, separators=(",", ":"))) for m in message.tool_calls or []
            )
        elif isinstance(message, AssistantToolCallMessage):
            char_count = len(message.content)
        return round(char_count / self.APPROXIMATE_TOKEN_LENGTH)

    def _get_estimated_langchain_message_tokens(self, message: BaseMessage) -> int:
        """
        Estimate token count for a message using character/4 heuristic.
        """
        char_count = 0
        if isinstance(message.content, str):
            char_count = len(message.content)
        else:
            for content in message.content:
                if isinstance(content, str):
                    char_count += len(content)
                elif isinstance(content, dict):
                    char_count += self._count_json_tokens(content)
        if isinstance(message, LangchainAIMessage) and message.tool_calls:
            for tool_call in message.tool_calls:
                char_count += len(json.dumps(tool_call, separators=(",", ":")))
        return round(char_count / self.APPROXIMATE_TOKEN_LENGTH)

    def _get_conversation_window(self, messages: Sequence[T], start_id: str) -> Sequence[T]:
        """
        Get messages from the start_id onwards.
        """
        for idx, message in enumerate(messages):
            if message.id == start_id:
                return messages[idx:]
        return messages

    def _get_estimated_tools_tokens(self, tools: LangchainTools) -> int:
        """
        Estimate token count for tools by converting them to JSON schemas.
        """
        if not tools:
            return 0

        total_chars = 0
        for tool in tools:
            tool_schema = convert_to_openai_tool(tool)
            total_chars += self._count_json_tokens(tool_schema)
        return round(total_chars / self.APPROXIMATE_TOKEN_LENGTH)

    def _count_json_tokens(self, json_data: dict) -> int:
        return len(json.dumps(json_data, separators=(",", ":")))

    @abstractmethod
    async def _get_token_count(
        self,
        model: Any,
        messages: list[BaseMessage],
        tools: LangchainTools | None = None,
        thinking_config: dict[str, Any] | None = None,
        **kwargs,
    ) -> int:
        raise NotImplementedError

    def _get_mode_message_with_context(
        self,
        window_messages: Sequence[AssistantMessageUnion],
        all_messages: Sequence[AssistantMessageUnion],
        agent_mode: AgentMode,
    ) -> ContextMessage | None:
        """
        Get mode reminder message with context-aware checking.
        Only injects a reminder if the mode is not evident in the window.
        """
        # Check if initial mode message exists in the window (not full history,
        # since compaction may have moved it outside the window)
        if self._has_initial_mode_message(window_messages):
            return None
        # Check if mode is evident in the current window
        if self._is_mode_evident_in_window(window_messages):
            return None
        return ContextMessage(
            content=ROOT_AGENT_MODE_REMINDER_PROMPT.format(mode=agent_mode.value),
            id=str(uuid4()),
        )

    def _should_add_mode_reminder(self, messages: Sequence[AssistantMessageUnion]) -> bool:
        """
        Determine if a mode reminder should be added to the messages.
        Returns True if:
        - agent_mode is set
        - mode is not evident in the messages (no switch_mode call for current mode)
        - initial mode message is not present in the messages
        """
        if self._has_initial_mode_message(messages):
            return False
        if self._is_mode_evident_in_window(messages):
            return False
        return True

    def _is_mode_evident_in_window(self, messages: Sequence[AssistantMessageUnion]) -> bool:
        """
        Check if the current agent mode is evident in the conversation window.
        Returns True if there's a switch_mode tool call for the current mode in the messages.
        """

        for message in messages:
            if isinstance(message, AssistantMessage) and message.tool_calls:
                for tool_call in message.tool_calls:
                    if tool_call.name == AssistantTool.SWITCH_MODE:
                        return True
        return False

    def _has_initial_mode_message(self, messages: Sequence[AssistantMessageUnion]) -> bool:
        """
        Check if the initial mode message from the context manager is present in the messages.
        """
        for message in messages:
            if isinstance(message, ContextMessage) and CONTEXT_INITIAL_MODE_PROMPT in message.content:
                return True
        return False

    def _find_last_todo_write_message(self, messages: Sequence[T]) -> AssistantMessage | None:
        """
        Find the last AssistantMessage with a TODO_WRITE tool call.
        Searches backwards through ALL messages (not just window).

        Returns:
            The last AssistantMessage containing a TODO_WRITE tool call, or None.
        """
        for message in reversed(messages):
            if isinstance(message, AssistantMessage) and message.tool_calls:
                for tool_call in message.tool_calls:
                    if tool_call.name == AssistantTool.TODO_WRITE:
                        return message
        return None

    def _is_todo_in_window(self, todo_message: AssistantMessage, window_messages: Sequence[T]) -> bool:
        """
        Check if the todo message is present in the given window.

        Args:
            todo_message: The AssistantMessage containing TODO_WRITE tool call
            window_messages: Messages in the current window

        Returns:
            True if todo_message is found in window_messages (by ID or object identity)
        """
        # Check by ID if available
        if todo_message.id:
            return any(msg.id == todo_message.id for msg in window_messages)

        # Fall back to object identity check if no ID
        return any(msg is todo_message for msg in window_messages)

    def _get_todo_reminder_message(self, messages: Sequence[T], window_messages: Sequence[T]) -> HumanMessage | None:
        """
        Create a todo reminder message if:
        1. A TODO_WRITE tool call exists in the conversation
        2. That todo message is NOT in the new window

        Args:
            messages: All messages (for finding the last todo)
            window_messages: Messages in the new window (for checking if todo is present)

        Returns:
            HumanMessage containing the todo reminder, or None if no reminder needed
        """
        # Find the last TODO_WRITE tool call
        todo_message = self._find_last_todo_write_message(messages)
        if not todo_message:
            return None

        # Check if it's already in the window
        if self._is_todo_in_window(todo_message, window_messages):
            return None

        # Extract the todo list from the tool call
        if not todo_message.tool_calls:
            return None

        todo_tool_call = next(
            (tc for tc in todo_message.tool_calls if tc.name == AssistantTool.TODO_WRITE),
            None,
        )
        if not todo_tool_call:
            return None

        # Format the reminder message using TodoWriteTool
        try:
            todo_content = TodoWriteTool.format_todo_list(todo_tool_call.args)
        except ValidationError:
            return None

        reminder_content = format_prompt_string(ROOT_TODO_REMINDER_PROMPT, todo_content=todo_content)
        return HumanMessage(content=reminder_content, id=str(uuid4()))


class AnthropicConversationCompactionManager(ConversationCompactionManager):
    async def _get_token_count(
        self,
        model: ChatAnthropic,
        messages: list[BaseMessage],
        tools: LangchainTools | None = None,
        thinking_config: dict[str, Any] | None = None,
        **kwargs,
    ) -> int:
        return await database_sync_to_async(model.get_num_tokens_from_messages, thread_sensitive=False)(
            messages, thinking=thinking_config, tools=tools
        )