Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

40.4 KB
Raw Blame
from collections.abc import AsyncIterator, Callable
from contextlib import asynccontextmanager
from typing import cast
from uuid import uuid4

from posthog.test.base import BaseTest
from unittest.mock import AsyncMock, MagicMock, patch

import httpx
import openai
import anthropic
from langgraph.checkpoint.memory import MemorySaver
from langgraph.graph import END, START, StateGraph
from langgraph.types import interrupt
from parameterized import parameterized

from posthog.schema import AssistantEventType, FailureMessage, HumanMessage

from products.posthog_ai.backend.models.assistant import Conversation

from ee.hogai.core.base import BaseAssistantGraph
from ee.hogai.tool import ApprovalRequest, ClientToolCallRequest
from ee.hogai.utils.types.base import AssistantState, ConversationTitleAction, PartialAssistantState


@asynccontextmanager
async def mock_lock_conversation():
    yield


async def _async_generator_that_raises(exception: Exception) -> AsyncIterator[None]:
    raise exception
    yield  # type: ignore[unreachable]


class TestRunnerLLMProviderErrorHandling(BaseTest):
    def setUp(self):
        super().setUp()
        self.conversation = Conversation.objects.create(team=self.team, user=self.user)

    def _create_mock_runner(self, exception_to_raise):
        """Create a mock runner with all necessary dependencies."""
        from ee.hogai.core.runner import BaseAgentRunner

        mock_graph = MagicMock()
        mock_graph.astream = MagicMock(return_value=_async_generator_that_raises(exception_to_raise))
        mock_graph.aget_state = AsyncMock(return_value=MagicMock(values={}, next=None))
        mock_graph.aupdate_state = AsyncMock()

        mock_stream_processor = MagicMock()
        mock_stream_processor.mark_id_as_streamed = MagicMock()

        # Create a proper mock graph class that returns a mock when instantiated
        mock_graph_class = MagicMock()
        mock_graph_instance = MagicMock()
        mock_graph_instance.compile_full_graph = MagicMock(return_value=mock_graph)
        mock_graph_class.return_value = mock_graph_instance

        class TestRunner(BaseAgentRunner):
            def get_initial_state(self):
                return AssistantState(messages=[])

            def get_resumed_state(self):
                return PartialAssistantState(messages=[])

        runner = TestRunner(
            team=self.team,
            conversation=self.conversation,
            user=self.user,
            graph_class=cast(type[BaseAssistantGraph], mock_graph_class),
            state_type=AssistantState,
            partial_state_type=PartialAssistantState,
            stream_processor=mock_stream_processor,
        )
        runner._graph = mock_graph

        return runner, mock_graph

    @parameterized.expand(
        [
            (
                "anthropic_bad_request",
                anthropic.BadRequestError(
                    message="prompt is too long: 1004486 tokens > 1000000 maximum",
                    response=httpx.Response(
                        status_code=400, request=httpx.Request("POST", "https://api.anthropic.com/v1/messages")
                    ),
                    body={"type": "error", "error": {"type": "invalid_request_error"}},
                ),
                "anthropic",
            ),
            (
                "openai_bad_request",
                openai.BadRequestError(
                    message="This model's maximum context length is 128000 tokens.",
                    response=httpx.Response(
                        status_code=400, request=httpx.Request("POST", "https://api.openai.com/v1/chat/completions")
                    ),
                    body=None,
                ),
                "openai",
            ),
        ]
    )
    async def test_llm_client_errors_handled_with_appropriate_message(self, _name, exception, expected_provider):
        """Client errors (400, 422) get a user-actionable message, not 'try again later'."""
        runner, mock_graph = self._create_mock_runner(exception)

        with (
            patch.object(runner, "_init_or_update_state", new_callable=AsyncMock, return_value=None),
            patch.object(runner, "_lock_conversation", return_value=mock_lock_conversation()),
            patch("ee.hogai.core.runner.LLM_CLIENT_ERROR_COUNTER") as mock_counter,
            patch("ee.hogai.core.runner.posthoganalytics") as mock_posthog,
            patch("ee.hogai.core.runner.logger") as mock_logger,
        ):
            results = []
            async for event_type, message in runner.astream(
                stream_message_chunks=False, stream_first_message=False, stream_only_assistant_messages=True
            ):
                results.append((event_type, message))

            # Verify that a FailureMessage was yielded with appropriate message
            self.assertEqual(len(results), 1)
            event_type, message = results[0]
            self.assertEqual(event_type, AssistantEventType.MESSAGE)
            self.assertIsInstance(message, FailureMessage)
            self.assertEqual(
                message.content,
                "I'm unable to process this request. The conversation may be too long. Please start a new conversation.",
            )

            # Verify state was reset
            mock_graph.aupdate_state.assert_called()

            # Verify Prometheus counter was incremented with correct provider
            mock_counter.labels.assert_called_with(provider=expected_provider)
            mock_counter.labels.return_value.inc.assert_called_once()

            # Verify error was logged as client error
            mock_logger.exception.assert_called_once()
            call_args = mock_logger.exception.call_args
            self.assertEqual(call_args[0][0], "llm_client_error")
            self.assertEqual(call_args[1]["provider"], expected_provider)

            # Verify exception was captured with correct type
            mock_posthog.capture_exception.assert_called_once()
            capture_call_args = mock_posthog.capture_exception.call_args
            self.assertEqual(capture_call_args[1]["properties"]["error_type"], "llm_client_error")
            self.assertEqual(capture_call_args[1]["properties"]["provider"], expected_provider)

    @parameterized.expand(
        [
            (
                "anthropic_internal_server_error",
                anthropic.InternalServerError(
                    message="Internal server error",
                    response=httpx.Response(
                        status_code=500, request=httpx.Request("POST", "https://api.anthropic.com/v1/messages")
                    ),
                    body=None,
                ),
                "anthropic",
            ),
            (
                "anthropic_rate_limit",
                anthropic.RateLimitError(
                    message="Rate limit exceeded",
                    response=httpx.Response(
                        status_code=429, request=httpx.Request("POST", "https://api.anthropic.com/v1/messages")
                    ),
                    body=None,
                ),
                "anthropic",
            ),
            (
                "openai_internal_server_error",
                openai.InternalServerError(
                    message="Internal server error",
                    response=httpx.Response(
                        status_code=500, request=httpx.Request("POST", "https://api.openai.com/v1/chat/completions")
                    ),
                    body=None,
                ),
                "openai",
            ),
            (
                "openai_rate_limit",
                openai.RateLimitError(
                    message="Rate limit exceeded",
                    response=httpx.Response(
                        status_code=429, request=httpx.Request("POST", "https://api.openai.com/v1/chat/completions")
                    ),
                    body=None,
                ),
                "openai",
            ),
        ]
    )
    async def test_llm_transient_errors_handled_with_retry_message(self, _name, exception, expected_provider):
        """Transient errors (5xx, rate limits) get a 'try again later' message."""
        runner, mock_graph = self._create_mock_runner(exception)

        with (
            patch.object(runner, "_init_or_update_state", new_callable=AsyncMock, return_value=None),
            patch.object(runner, "_lock_conversation", return_value=mock_lock_conversation()),
            patch("ee.hogai.core.runner.LLM_PROVIDER_ERROR_COUNTER") as mock_counter,
            patch("ee.hogai.core.runner.posthoganalytics") as mock_posthog,
            patch("ee.hogai.core.runner.logger") as mock_logger,
        ):
            results = []
            async for event_type, message in runner.astream(
                stream_message_chunks=False, stream_first_message=False, stream_only_assistant_messages=True
            ):
                results.append((event_type, message))

            # Verify that a FailureMessage was yielded with retry message
            self.assertEqual(len(results), 1)
            event_type, message = results[0]
            self.assertEqual(event_type, AssistantEventType.MESSAGE)
            self.assertIsInstance(message, FailureMessage)
            self.assertEqual(
                message.content,
                "I'm unable to respond right now due to a temporary service issue. Please try again later.",
            )

            # Verify state was reset
            mock_graph.aupdate_state.assert_called()

            # Verify Prometheus counter was incremented with correct provider
            mock_counter.labels.assert_called_with(provider=expected_provider)
            mock_counter.labels.return_value.inc.assert_called_once()

            # Verify error was logged as provider error
            mock_logger.exception.assert_called_once()
            call_args = mock_logger.exception.call_args
            self.assertEqual(call_args[0][0], "llm_provider_error")
            self.assertEqual(call_args[1]["provider"], expected_provider)

            # Verify exception was captured with correct type
            mock_posthog.capture_exception.assert_called_once()
            capture_call_args = mock_posthog.capture_exception.call_args
            self.assertEqual(capture_call_args[1]["properties"]["error_type"], "llm_provider_error")
            self.assertEqual(capture_call_args[1]["properties"]["provider"], expected_provider)

    @parameterized.expand(
        [
            (
                "httpx_read_error",
                httpx.ReadError("Connection reset by peer"),
                "ReadError",
            ),
            (
                "httpx_connect_error",
                httpx.ConnectError("Connection refused"),
                "ConnectError",
            ),
        ]
    )
    async def test_httpx_transport_errors_tracked_separately_from_provider_errors(
        self, _name, exception, expected_error_type
    ):
        """httpx transport errors are network issues, not LLM provider errors, and use a separate counter."""
        runner, mock_graph = self._create_mock_runner(exception)

        with (
            patch.object(runner, "_init_or_update_state", new_callable=AsyncMock, return_value=None),
            patch.object(runner, "_lock_conversation", return_value=mock_lock_conversation()),
            patch("ee.hogai.core.runner.LLM_TRANSPORT_ERROR_COUNTER") as mock_transport_counter,
            patch("ee.hogai.core.runner.LLM_PROVIDER_ERROR_COUNTER") as mock_provider_counter,
            patch("ee.hogai.core.runner.posthoganalytics") as mock_posthog,
            patch("ee.hogai.core.runner.logger") as mock_logger,
        ):
            results = []
            async for event_type, message in runner.astream(
                stream_message_chunks=False, stream_first_message=False, stream_only_assistant_messages=True
            ):
                results.append((event_type, message))

            self.assertEqual(len(results), 1)
            event_type, message = results[0]
            self.assertEqual(event_type, AssistantEventType.MESSAGE)
            self.assertIsInstance(message, FailureMessage)
            self.assertEqual(
                message.content,
                "I'm unable to respond right now due to a temporary service issue. Please try again later.",
            )

            mock_graph.aupdate_state.assert_called()

            # Transport counter is incremented with error_type (not provider)
            mock_transport_counter.labels.assert_called_with(error_type=expected_error_type)
            mock_transport_counter.labels.return_value.inc.assert_called_once()

            # Provider counter is NOT incremented
            mock_provider_counter.labels.assert_not_called()

            # Logged as transport error, not provider error
            mock_logger.exception.assert_called_once()
            call_args = mock_logger.exception.call_args
            self.assertEqual(call_args[0][0], "llm_transport_error")
            self.assertEqual(call_args[1]["error_type"], expected_error_type)

            # Captured with transport error type, no provider field
            mock_posthog.capture_exception.assert_called_once()
            capture_call_args = mock_posthog.capture_exception.call_args
            self.assertEqual(capture_call_args[1]["properties"]["error_type"], "llm_transport_error")
            self.assertNotIn("provider", capture_call_args[1]["properties"])

    @parameterized.expand(
        [
            (
                "openai_generic_api_error",
                openai.APIError(
                    message="Unknown API error",
                    request=httpx.Request("POST", "https://api.openai.com/v1/chat/completions"),
                    body=None,
                ),
                "openai",
            ),
        ]
    )
    async def test_llm_generic_api_errors_handled_gracefully(self, _name, exception, expected_provider):
        """Generic API errors (not client or transient) get a generic message."""
        runner, mock_graph = self._create_mock_runner(exception)

        with (
            patch.object(runner, "_init_or_update_state", new_callable=AsyncMock, return_value=None),
            patch.object(runner, "_lock_conversation", return_value=mock_lock_conversation()),
            patch("ee.hogai.core.runner.LLM_PROVIDER_ERROR_COUNTER") as mock_counter,
            patch("ee.hogai.core.runner.posthoganalytics") as mock_posthog,
            patch("ee.hogai.core.runner.logger") as mock_logger,
        ):
            results = []
            async for event_type, message in runner.astream(
                stream_message_chunks=False, stream_first_message=False, stream_only_assistant_messages=True
            ):
                results.append((event_type, message))

            # Verify that a FailureMessage was yielded
            self.assertEqual(len(results), 1)
            event_type, message = results[0]
            self.assertEqual(event_type, AssistantEventType.MESSAGE)
            self.assertIsInstance(message, FailureMessage)
            self.assertEqual(
                message.content,
                "I'm unable to respond right now. Please try again later.",
            )

            # Verify state was reset
            mock_graph.aupdate_state.assert_called()

            # Verify Prometheus counter was incremented with correct provider
            mock_counter.labels.assert_called_with(provider=expected_provider)
            mock_counter.labels.return_value.inc.assert_called_once()

            # Verify error was logged as api error
            mock_logger.exception.assert_called_once()
            call_args = mock_logger.exception.call_args
            self.assertEqual(call_args[0][0], "llm_api_error")
            self.assertEqual(call_args[1]["provider"], expected_provider)

            # Verify exception was captured with correct type
            mock_posthog.capture_exception.assert_called_once()
            capture_call_args = mock_posthog.capture_exception.call_args
            self.assertEqual(capture_call_args[1]["properties"]["error_type"], "llm_api_error")
            self.assertEqual(capture_call_args[1]["properties"]["provider"], expected_provider)


class TestRunnerSubagentBehavior(BaseTest):
    """
    Tests for subagent-specific behavior when use_checkpointer=False.

    Subagents differ from the main agent in several ways:
    - They skip interrupt handling
    - They don't reset state on errors
    - They don't lock the conversation
    - They skip checkpoint-related state initialization
    """

    def setUp(self):
        super().setUp()
        self.conversation = Conversation.objects.create(team=self.team, user=self.user)

    def _create_runner(self, use_checkpointer: bool, graph_mock=None):
        """Create a runner with specified checkpointer setting."""
        from ee.hogai.core.runner import BaseAgentRunner

        mock_graph = graph_mock or MagicMock()
        if not graph_mock:
            mock_graph.aget_state = AsyncMock(return_value=MagicMock(values={}, next=None))
            mock_graph.aupdate_state = AsyncMock()

        mock_stream_processor = MagicMock()
        mock_stream_processor.mark_id_as_streamed = MagicMock()

        class TestRunner(BaseAgentRunner):
            def get_initial_state(self):
                return AssistantState(messages=[])

            def get_resumed_state(self):
                return PartialAssistantState(messages=[])

        runner = TestRunner(
            team=self.team,
            conversation=self.conversation,
            user=self.user,
            graph_class=cast(type[BaseAssistantGraph], mock_graph),
            state_type=AssistantState,
            partial_state_type=PartialAssistantState,
            stream_processor=mock_stream_processor,
            use_checkpointer=use_checkpointer,
        )
        runner._graph = mock_graph

        return runner, mock_graph

    @parameterized.expand(
        [
            ("with_checkpointer", True, True),
            ("without_checkpointer", False, False),
        ]
    )
    async def test_interrupt_handling_depends_on_checkpointer(self, _name, use_checkpointer, should_check_interrupts):
        """
        When use_checkpointer=False (subagent), the runner should skip interrupt handling
        and return early after streaming.
        """

        async def empty_stream():
            yield
            return

        runner, mock_graph = self._create_runner(use_checkpointer)
        mock_graph.astream = MagicMock(return_value=empty_stream())

        # Set up aget_state to return a state with pending next steps (interrupt)
        mock_state = MagicMock()
        mock_state.values = {}
        mock_state.next = ["some_node"]  # Indicates pending interrupt
        mock_state.tasks = []
        mock_graph.aget_state = AsyncMock(return_value=mock_state)

        with (
            patch.object(runner, "_init_or_update_state", new_callable=AsyncMock, return_value=None),
            patch.object(runner, "_lock_conversation", return_value=mock_lock_conversation()),
        ):
            results = []
            async for event_type, message in runner.astream(
                stream_message_chunks=False, stream_first_message=False, stream_only_assistant_messages=True
            ):
                results.append((event_type, message))

        if should_check_interrupts:
            mock_graph.aget_state.assert_called()
        else:
            mock_graph.aget_state.assert_not_called()

    @parameterized.expand(
        [
            ("with_checkpointer", True, True),
            ("without_checkpointer", False, False),
        ]
    )
    async def test_state_reset_on_llm_error_depends_on_checkpointer(self, _name, use_checkpointer, should_reset_state):
        """
        When use_checkpointer=False (subagent), state should NOT be reset on LLM errors.
        """
        exception = anthropic.InternalServerError(
            message="Internal server error",
            response=httpx.Response(
                status_code=500, request=httpx.Request("POST", "https://api.anthropic.com/v1/messages")
            ),
            body=None,
        )

        runner, mock_graph = self._create_runner(use_checkpointer)
        mock_graph.astream = MagicMock(return_value=_async_generator_that_raises(exception))

        with (
            patch.object(runner, "_init_or_update_state", new_callable=AsyncMock, return_value=None),
            patch.object(runner, "_lock_conversation", return_value=mock_lock_conversation()),
            patch("ee.hogai.core.runner.LLM_PROVIDER_ERROR_COUNTER"),
            patch("ee.hogai.core.runner.posthoganalytics"),
            patch("ee.hogai.core.runner.logger"),
        ):
            results = []
            async for event_type, message in runner.astream(
                stream_message_chunks=False, stream_first_message=False, stream_only_assistant_messages=True
            ):
                results.append((event_type, message))

            # Both should yield a failure message
            self.assertEqual(len(results), 1)
            self.assertIsInstance(results[0][1], FailureMessage)

            # Only checkpointer mode should reset state
            if should_reset_state:
                mock_graph.aupdate_state.assert_called()
            else:
                mock_graph.aupdate_state.assert_not_called()

    @parameterized.expand(
        [
            ("with_checkpointer", True, True),
            ("without_checkpointer", False, False),
        ]
    )
    async def test_state_reset_on_general_exception_depends_on_checkpointer(
        self, _name, use_checkpointer, should_reset_state
    ):
        """
        When use_checkpointer=False (subagent), state should NOT be reset on general exceptions.
        """
        exception = RuntimeError("Unexpected error")

        runner, mock_graph = self._create_runner(use_checkpointer)
        mock_graph.astream = MagicMock(return_value=_async_generator_that_raises(exception))

        # Set up state for checkpointer case
        mock_state = MagicMock()
        mock_state.values = {"messages": []}
        mock_state.next = None
        mock_graph.aget_state = AsyncMock(return_value=mock_state)

        with (
            patch.object(runner, "_init_or_update_state", new_callable=AsyncMock, return_value=None),
            patch.object(runner, "_lock_conversation", return_value=mock_lock_conversation()),
            patch("ee.hogai.core.runner.posthoganalytics"),
            patch("ee.hogai.core.runner.logger"),
        ):
            results = []
            async for event_type, message in runner.astream(
                stream_message_chunks=False, stream_first_message=False, stream_only_assistant_messages=True
            ):
                results.append((event_type, message))

            if should_reset_state:
                mock_graph.aupdate_state.assert_called()
            else:
                mock_graph.aupdate_state.assert_not_called()

    async def test_subagent_lock_conversation_does_not_update_status(self):
        """
        When use_checkpointer=False (subagent), _lock_conversation should yield
        immediately without updating conversation status.
        """
        runner, _ = self._create_runner(use_checkpointer=False)

        # Capture initial status
        initial_status = self.conversation.status

        async with runner._lock_conversation():
            # Status should not have changed during the context
            await self.conversation.arefresh_from_db()
            self.assertEqual(self.conversation.status, initial_status)

        # Status should still be unchanged after exiting
        await self.conversation.arefresh_from_db()
        self.assertEqual(self.conversation.status, initial_status)

    async def test_main_agent_lock_conversation_updates_status(self):
        """
        When use_checkpointer=True (main agent), _lock_conversation should
        update conversation status to IN_PROGRESS and then to IDLE.
        """
        runner, _ = self._create_runner(use_checkpointer=True)

        # Ensure conversation starts as IDLE
        self.conversation.status = Conversation.Status.IDLE
        await self.conversation.asave()

        async with runner._lock_conversation():
            await self.conversation.arefresh_from_db()
            self.assertEqual(self.conversation.status, Conversation.Status.IN_PROGRESS)

        await self.conversation.arefresh_from_db()
        self.assertEqual(self.conversation.status, Conversation.Status.IDLE)

    @parameterized.expand(
        [
            ("with_checkpointer", True, True),
            ("without_checkpointer", False, False),
        ]
    )
    async def test_init_or_update_state_checkpoint_lookup_depends_on_checkpointer(
        self, _name, use_checkpointer, should_check_checkpoint
    ):
        """
        When use_checkpointer=False (subagent), _init_or_update_state should skip
        checkpoint state lookup and go directly to initial state.
        """
        runner, mock_graph = self._create_runner(use_checkpointer)

        mock_state = MagicMock()
        mock_state.values = {"messages": []}
        mock_state.next = None
        mock_graph.aget_state = AsyncMock(return_value=mock_state)

        with patch("ee.hogai.core.runner.is_cloud", return_value=False):
            await runner._init_or_update_state()

        if should_check_checkpoint:
            mock_graph.aget_state.assert_called()
        else:
            mock_graph.aget_state.assert_not_called()

    async def test_subagent_callback_handler_uses_parent_span_id(self):
        """
        When parent_span_id is provided, SubagentCallbackHandler should be used
        to nest all events under the parent span.
        """
        from uuid import uuid4

        from ee.hogai.core.runner import SubagentCallbackHandler

        parent_span_id = uuid4()

        with (
            patch("ee.hogai.core.runner.is_cloud", return_value=False),
            patch("ee.hogai.core.runner.posthoganalytics") as mock_posthog,
        ):
            mock_client = MagicMock()
            mock_posthog.default_client = mock_client

            from ee.hogai.core.runner import BaseAgentRunner

            mock_graph = MagicMock()
            mock_stream_processor = MagicMock()

            class TestRunner(BaseAgentRunner):
                def get_initial_state(self):
                    return AssistantState(messages=[])

                def get_resumed_state(self):
                    return PartialAssistantState(messages=[])

            runner = TestRunner(
                team=self.team,
                conversation=self.conversation,
                user=self.user,
                graph_class=cast(type[BaseAssistantGraph], mock_graph),
                state_type=AssistantState,
                partial_state_type=PartialAssistantState,
                stream_processor=mock_stream_processor,
                use_checkpointer=False,
                parent_span_id=parent_span_id,
            )

            # Check that SubagentCallbackHandler was used
            self.assertEqual(len(runner._callback_handlers), 1)
            self.assertIsInstance(runner._callback_handlers[0], SubagentCallbackHandler)
            handler = cast(SubagentCallbackHandler, runner._callback_handlers[0])
            self.assertEqual(handler._parent_span_id, parent_span_id)


class TestRunnerConversationTitleAction(BaseTest):
    def setUp(self):
        super().setUp()
        self.conversation = Conversation.objects.create(team=self.team, user=self.user)

    def _create_runner(self, stream_updates=None):
        from ee.hogai.core.runner import BaseAgentRunner

        async def _fake_astream(*args, **kwargs):
            for update in stream_updates or []:
                yield update

        mock_graph = MagicMock()
        mock_graph.astream = MagicMock(return_value=_fake_astream())
        mock_graph.aget_state = AsyncMock(return_value=MagicMock(values={}, next=None))
        mock_graph.aupdate_state = AsyncMock()

        mock_stream_processor = MagicMock()
        mock_stream_processor.mark_id_as_streamed = MagicMock()
        mock_stream_processor.process = AsyncMock(return_value=None)
        mock_stream_processor.process_langgraph_update = AsyncMock(return_value=None)

        mock_graph_class = MagicMock()
        mock_graph_instance = MagicMock()
        mock_graph_instance.compile_full_graph = MagicMock(return_value=mock_graph)
        mock_graph_class.return_value = mock_graph_instance

        class TestRunner(BaseAgentRunner):
            def get_initial_state(self):
                return AssistantState(messages=[])

            def get_resumed_state(self):
                return PartialAssistantState(messages=[])

        runner = TestRunner(
            team=self.team,
            conversation=self.conversation,
            user=self.user,
            graph_class=cast(type[BaseAssistantGraph], mock_graph_class),
            state_type=AssistantState,
            partial_state_type=PartialAssistantState,
            stream_processor=mock_stream_processor,
        )
        runner._graph = mock_graph

        return runner, mock_graph

    async def test_process_update_sets_conversation_title_and_pending_flag(self):
        runner, _ = self._create_runner()
        runner._pending_conversation_update = False

        # In the LangGraph custom stream, ConversationTitleAction arrives as (namespace, "custom", action)
        action = ConversationTitleAction(title="My Generated Title")
        raw_update = (("title_generator",), "custom", action)
        result = await runner._process_update(raw_update)

        self.assertIsNone(result)
        self.assertEqual(runner._conversation.title, "My Generated Title")
        self.assertTrue(runner._pending_conversation_update)

    async def test_process_update_does_not_set_pending_flag_for_other_updates(self):
        runner, _ = self._create_runner()
        runner._pending_conversation_update = False

        # A values update — (namespace, "values", state_dict) format from LangGraph multi-mode streaming
        other_update: tuple[tuple[str, ...], str, dict] = (("some_node",), "values", {})
        await runner._process_update(other_update)

        self.assertFalse(runner._pending_conversation_update)

    async def test_astream_yields_conversation_event_when_title_action_received(self):
        title_action = ConversationTitleAction(title="Streamed Title")
        # Wrap in the LangGraph custom stream tuple format
        raw_update = (("title_generator",), "custom", title_action)

        runner, mock_graph = self._create_runner(stream_updates=[raw_update])

        with (
            patch.object(runner, "_init_or_update_state", new_callable=AsyncMock, return_value=None),
            patch.object(runner, "_lock_conversation", return_value=mock_lock_conversation()),
        ):
            results = []
            async for event_type, payload in runner.astream(
                stream_message_chunks=False,
                stream_first_message=False,
                stream_only_assistant_messages=False,
            ):
                results.append((event_type, payload))

        conversation_events = [(et, p) for et, p in results if et == AssistantEventType.CONVERSATION]
        self.assertEqual(len(conversation_events), 1)
        _, conversation = conversation_events[0]
        self.assertEqual(conversation.title, "Streamed Title")


class TestRunnerClientToolCallInterrupt(BaseTest):
    """
    A ClientToolCallRequest interrupt must stream nothing and leave graph state untouched,
    so the frontend handler can resume it with a payload (or a new human message can abandon
    it and start fresh).
    """

    def setUp(self):
        super().setUp()
        self.conversation = Conversation.objects.create(team=self.team, user=self.user)

    def _create_runner_with_interrupt(self, interrupt_value):
        from ee.hogai.core.runner import BaseAgentRunner

        async def empty_stream():
            return
            yield

        mock_graph = MagicMock()
        mock_graph.astream = MagicMock(return_value=empty_stream())
        mock_state = MagicMock()
        mock_state.values = {}
        mock_state.next = ["pending_node"]
        mock_state.tasks = [MagicMock(result=None, interrupts=[MagicMock(value=interrupt_value)])]
        mock_graph.aget_state = AsyncMock(return_value=mock_state)
        mock_graph.aupdate_state = AsyncMock()

        mock_stream_processor = MagicMock()
        mock_stream_processor.mark_id_as_streamed = MagicMock()

        class TestRunner(BaseAgentRunner):
            def get_initial_state(self):
                return AssistantState(messages=[])

            def get_resumed_state(self):
                return PartialAssistantState(messages=[])

        runner = TestRunner(
            team=self.team,
            conversation=self.conversation,
            user=self.user,
            graph_class=cast(type[BaseAssistantGraph], mock_graph),
            state_type=AssistantState,
            partial_state_type=PartialAssistantState,
            stream_processor=mock_stream_processor,
            use_checkpointer=True,
        )
        runner._graph = mock_graph
        return runner, mock_graph

    async def _collect_message_events(self, runner):
        with (
            patch.object(runner, "_init_or_update_state", new_callable=AsyncMock, return_value=None),
            patch.object(runner, "_lock_conversation", return_value=mock_lock_conversation()),
        ):
            results = []
            async for event_type, payload in runner.astream(
                stream_message_chunks=False, stream_first_message=False, stream_only_assistant_messages=True
            ):
                results.append((event_type, payload))
        return [payload for event_type, payload in results if event_type == AssistantEventType.MESSAGE]

    async def test_streams_nothing_and_skips_state_update(self):
        runner, mock_graph = self._create_runner_with_interrupt(
            ClientToolCallRequest(tool_name="create_form", original_tool_call_id="call_1")
        )

        messages = await self._collect_message_events(runner)

        self.assertEqual(messages, [])
        mock_graph.aupdate_state.assert_not_called()

    async def test_string_interrupt_still_streams_message_and_updates_state(self):
        runner, mock_graph = self._create_runner_with_interrupt("Please clarify your request")

        messages = await self._collect_message_events(runner)

        self.assertTrue(any(getattr(m, "content", None) == "Please clarify your request" for m in messages))
        mock_graph.aupdate_state.assert_called_once()

    @parameterized.expand([("approve", None), ("reject", None), ("reject", "make them monthly")])
    async def test_response_targets_selected_parallel_approval(self, action: str, feedback: str | None) -> None:
        executed: list[str] = []

        def operation(name: str) -> Callable[[AssistantState], dict[str, object]]:
            proposal_id = str(uuid4())

            def run(state: AssistantState) -> dict[str, object]:
                response = interrupt(
                    ApprovalRequest(
                        proposal_id=proposal_id, tool_name=name, preview=name, payload={}, original_tool_call_id=name
                    )
                )
                if response["action"] == "approve":
                    executed.append(name)
                return {}

            return run

        builder = StateGraph(AssistantState)
        for name in ("first", "second"):
            builder.add_node(name, operation(name))
            builder.add_edge(START, name)
            builder.add_edge(name, END)
        graph = builder.compile(checkpointer=MemorySaver())
        runner, _ = self._create_runner_with_interrupt(None)
        runner._graph = graph
        config = runner._get_config()
        config["callbacks"] = []
        await graph.ainvoke(AssistantState(messages=[]), config)
        snapshot = await graph.aget_state(config)
        approvals = {pending.value.tool_name: pending.value for task in snapshot.tasks for pending in task.interrupts}
        for approval in approvals.values():
            self.conversation.approval_decisions[approval.proposal_id] = {"decision_status": "pending"}
        await self.conversation.asave(update_fields=["approval_decisions"])
        runner._resume_payload = {"action": action, "proposal_id": approvals["second"].proposal_id}
        if feedback:
            runner._resume_payload["feedback"] = feedback
        command = await runner._init_or_update_state()
        await graph.ainvoke(command, config)
        self.assertEqual(executed, ["second"] if action == "approve" else [])
        after = await graph.aget_state(config)
        remaining = [pending.value for task in after.tasks if task.result is None for pending in task.interrupts]
        if feedback:
            await self.conversation.arefresh_from_db()
            self.assertEqual(remaining, [])
            self.assertEqual(
                {
                    name: self.conversation.approval_decisions[approval.proposal_id]["decision_status"]
                    for name, approval in approvals.items()
                },
                {"first": "auto_rejected", "second": "rejected"},
            )
            return
        self.assertEqual([request.proposal_id for request in remaining], [approvals["first"].proposal_id])
        await self.conversation.arefresh_from_db()
        self.assertEqual(
            self.conversation.approval_decisions[approvals["first"].proposal_id]["decision_status"], "pending"
        )
        self.assertEqual(
            self.conversation.approval_decisions[approvals["second"].proposal_id]["decision_status"],
            "approved" if action == "approve" else "rejected",
        )
        with self.assertRaisesMessage(ValueError, "Approval does not match a pending operation"):
            await runner._init_or_update_state()
        runner._resume_payload = {"action": "reject", "proposal_id": approvals["first"].proposal_id}
        await graph.ainvoke(await runner._init_or_update_state(), config)
        self.assertFalse((await graph.aget_state(config)).next)
        self.assertEqual(executed, ["second"] if action == "approve" else [])

    @parameterized.expand(
        [
            ("new_message", {"live": "auto_rejected", "stale": "auto_rejected"}),
            ("stale_response", {"live": "pending", "stale": "auto_rejected"}),
            ("stale_response_with_message", {"live": "auto_rejected", "stale": "auto_rejected"}),
            ("live_response", {"live": "approved", "stale": "auto_rejected"}),
        ]
    )
    async def test_unresumable_pending_approval_is_auto_rejected(self, trigger: str, expected: dict[str, str]) -> None:
        live_id, stale_id = str(uuid4()), str(uuid4())

        def operation(state: AssistantState) -> dict[str, object]:
            interrupt(ApprovalRequest(proposal_id=live_id, tool_name="create_insight", preview="Update", payload={}))
            return {}

        builder = StateGraph(AssistantState)
        builder.add_node("operation", operation)
        builder.add_edge(START, "operation")
        builder.add_edge("operation", END)
        graph = builder.compile(checkpointer=MemorySaver())
        runner, _ = self._create_runner_with_interrupt(None)
        runner._graph = graph
        config = runner._get_config()
        config["callbacks"] = []
        await graph.ainvoke(AssistantState(messages=[]), config)
        self.conversation.approval_decisions = {
            live_id: {"decision_status": "pending"},
            stale_id: {"decision_status": "pending"},
        }
        await self.conversation.asave(update_fields=["approval_decisions"])

        if trigger == "new_message":
            runner._latest_message = HumanMessage(content="are you stuck?", id=str(uuid4()))
            await runner._init_or_update_state()
        elif trigger == "stale_response_with_message":
            runner._latest_message = HumanMessage(content="are you stuck?", id=str(uuid4()))
            runner._resume_payload = {"action": "reject", "proposal_id": stale_id}
            self.assertIsInstance(await runner._init_or_update_state(), AssistantState)
        elif trigger == "live_response":
            runner._resume_payload = {"action": "approve", "proposal_id": live_id}
            await runner._init_or_update_state()
        else:
            runner._resume_payload = {"action": "approve", "proposal_id": stale_id}
            with self.assertRaisesMessage(ValueError, "Approval does not match a pending operation"):
                await runner._init_or_update_state()

        await self.conversation.arefresh_from_db()
        self.assertEqual(
            {
                "live": self.conversation.approval_decisions[live_id]["decision_status"],
                "stale": self.conversation.approval_decisions[stale_id]["decision_status"],
            },
            expected,
        )