:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
from collections.abc import AsyncIterator, Callable
from contextlib import asynccontextmanager
from typing import cast
from uuid import uuid4
from posthog.test.base import BaseTest
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import openai
import anthropic
from langgraph.checkpoint.memory import MemorySaver
from langgraph.graph import END, START, StateGraph
from langgraph.types import interrupt
from parameterized import parameterized
from posthog.schema import AssistantEventType, FailureMessage, HumanMessage
from products.posthog_ai.backend.models.assistant import Conversation
from ee.hogai.core.base import BaseAssistantGraph
from ee.hogai.tool import ApprovalRequest, ClientToolCallRequest
from ee.hogai.utils.types.base import AssistantState, ConversationTitleAction, PartialAssistantState
@asynccontextmanager
async def mock_lock_conversation():
yield
async def _async_generator_that_raises(exception: Exception) -> AsyncIterator[None]:
raise exception
yield # type: ignore[unreachable]
class TestRunnerLLMProviderErrorHandling(BaseTest):
def setUp(self):
super().setUp()
self.conversation = Conversation.objects.create(team=self.team, user=self.user)
def _create_mock_runner(self, exception_to_raise):
"""Create a mock runner with all necessary dependencies."""
from ee.hogai.core.runner import BaseAgentRunner
mock_graph = MagicMock()
mock_graph.astream = MagicMock(return_value=_async_generator_that_raises(exception_to_raise))
mock_graph.aget_state = AsyncMock(return_value=MagicMock(values={}, next=None))
mock_graph.aupdate_state = AsyncMock()
mock_stream_processor = MagicMock()
mock_stream_processor.mark_id_as_streamed = MagicMock()
# Create a proper mock graph class that returns a mock when instantiated
mock_graph_class = MagicMock()
mock_graph_instance = MagicMock()
mock_graph_instance.compile_full_graph = MagicMock(return_value=mock_graph)
mock_graph_class.return_value = mock_graph_instance
class TestRunner(BaseAgentRunner):
def get_initial_state(self):
return AssistantState(messages=[])
def get_resumed_state(self):
return PartialAssistantState(messages=[])
runner = TestRunner(
team=self.team,
conversation=self.conversation,
user=self.user,
graph_class=cast(type[BaseAssistantGraph], mock_graph_class),
state_type=AssistantState,
partial_state_type=PartialAssistantState,
stream_processor=mock_stream_processor,
)
runner._graph = mock_graph
return runner, mock_graph
@parameterized.expand(
[
(
"anthropic_bad_request",
anthropic.BadRequestError(
message="prompt is too long: 1004486 tokens > 1000000 maximum",
response=httpx.Response(
status_code=400, request=httpx.Request("POST", "https://api.anthropic.com/v1/messages")
),
body={"type": "error", "error": {"type": "invalid_request_error"}},
),
"anthropic",
),
(
"openai_bad_request",
openai.BadRequestError(
message="This model's maximum context length is 128000 tokens.",
response=httpx.Response(
status_code=400, request=httpx.Request("POST", "https://api.openai.com/v1/chat/completions")
),
body=None,
),
"openai",
),
]
)
async def test_llm_client_errors_handled_with_appropriate_message(self, _name, exception, expected_provider):
"""Client errors (400, 422) get a user-actionable message, not 'try again later'."""
runner, mock_graph = self._create_mock_runner(exception)
with (
patch.object(runner, "_init_or_update_state", new_callable=AsyncMock, return_value=None),
patch.object(runner, "_lock_conversation", return_value=mock_lock_conversation()),
patch("ee.hogai.core.runner.LLM_CLIENT_ERROR_COUNTER") as mock_counter,
patch("ee.hogai.core.runner.posthoganalytics") as mock_posthog,
patch("ee.hogai.core.runner.logger") as mock_logger,
):
results = []
async for event_type, message in runner.astream(
stream_message_chunks=False, stream_first_message=False, stream_only_assistant_messages=True
):
results.append((event_type, message))
# Verify that a FailureMessage was yielded with appropriate message
self.assertEqual(len(results), 1)
event_type, message = results[0]
self.assertEqual(event_type, AssistantEventType.MESSAGE)
self.assertIsInstance(message, FailureMessage)
self.assertEqual(
message.content,
"I'm unable to process this request. The conversation may be too long. Please start a new conversation.",
)
# Verify state was reset
mock_graph.aupdate_state.assert_called()
# Verify Prometheus counter was incremented with correct provider
mock_counter.labels.assert_called_with(provider=expected_provider)
mock_counter.labels.return_value.inc.assert_called_once()
# Verify error was logged as client error
mock_logger.exception.assert_called_once()
call_args = mock_logger.exception.call_args
self.assertEqual(call_args[0][0], "llm_client_error")
self.assertEqual(call_args[1]["provider"], expected_provider)
# Verify exception was captured with correct type
mock_posthog.capture_exception.assert_called_once()
capture_call_args = mock_posthog.capture_exception.call_args
self.assertEqual(capture_call_args[1]["properties"]["error_type"], "llm_client_error")
self.assertEqual(capture_call_args[1]["properties"]["provider"], expected_provider)
@parameterized.expand(
[
(
"anthropic_internal_server_error",
anthropic.InternalServerError(
message="Internal server error",
response=httpx.Response(
status_code=500, request=httpx.Request("POST", "https://api.anthropic.com/v1/messages")
),
body=None,
),
"anthropic",
),
(
"anthropic_rate_limit",
anthropic.RateLimitError(
message="Rate limit exceeded",
response=httpx.Response(
status_code=429, request=httpx.Request("POST", "https://api.anthropic.com/v1/messages")
),
body=None,
),
"anthropic",
),
(
"openai_internal_server_error",
openai.InternalServerError(
message="Internal server error",
response=httpx.Response(
status_code=500, request=httpx.Request("POST", "https://api.openai.com/v1/chat/completions")
),
body=None,
),
"openai",
),
(
"openai_rate_limit",
openai.RateLimitError(
message="Rate limit exceeded",
response=httpx.Response(
status_code=429, request=httpx.Request("POST", "https://api.openai.com/v1/chat/completions")
),
body=None,
),
"openai",
),
]
)
async def test_llm_transient_errors_handled_with_retry_message(self, _name, exception, expected_provider):
"""Transient errors (5xx, rate limits) get a 'try again later' message."""
runner, mock_graph = self._create_mock_runner(exception)
with (
patch.object(runner, "_init_or_update_state", new_callable=AsyncMock, return_value=None),
patch.object(runner, "_lock_conversation", return_value=mock_lock_conversation()),
patch("ee.hogai.core.runner.LLM_PROVIDER_ERROR_COUNTER") as mock_counter,
patch("ee.hogai.core.runner.posthoganalytics") as mock_posthog,
patch("ee.hogai.core.runner.logger") as mock_logger,
):
results = []
async for event_type, message in runner.astream(
stream_message_chunks=False, stream_first_message=False, stream_only_assistant_messages=True
):
results.append((event_type, message))
# Verify that a FailureMessage was yielded with retry message
self.assertEqual(len(results), 1)
event_type, message = results[0]
self.assertEqual(event_type, AssistantEventType.MESSAGE)
self.assertIsInstance(message, FailureMessage)
self.assertEqual(
message.content,
"I'm unable to respond right now due to a temporary service issue. Please try again later.",
)
# Verify state was reset
mock_graph.aupdate_state.assert_called()
# Verify Prometheus counter was incremented with correct provider
mock_counter.labels.assert_called_with(provider=expected_provider)
mock_counter.labels.return_value.inc.assert_called_once()
# Verify error was logged as provider error
mock_logger.exception.assert_called_once()
call_args = mock_logger.exception.call_args
self.assertEqual(call_args[0][0], "llm_provider_error")
self.assertEqual(call_args[1]["provider"], expected_provider)
# Verify exception was captured with correct type
mock_posthog.capture_exception.assert_called_once()
capture_call_args = mock_posthog.capture_exception.call_args
self.assertEqual(capture_call_args[1]["properties"]["error_type"], "llm_provider_error")
self.assertEqual(capture_call_args[1]["properties"]["provider"], expected_provider)
@parameterized.expand(
[
(
"httpx_read_error",
httpx.ReadError("Connection reset by peer"),
"ReadError",
),
(
"httpx_connect_error",
httpx.ConnectError("Connection refused"),
"ConnectError",
),
]
)
async def test_httpx_transport_errors_tracked_separately_from_provider_errors(
self, _name, exception, expected_error_type
):
"""httpx transport errors are network issues, not LLM provider errors, and use a separate counter."""
runner, mock_graph = self._create_mock_runner(exception)
with (
patch.object(runner, "_init_or_update_state", new_callable=AsyncMock, return_value=None),
patch.object(runner, "_lock_conversation", return_value=mock_lock_conversation()),
patch("ee.hogai.core.runner.LLM_TRANSPORT_ERROR_COUNTER") as mock_transport_counter,
patch("ee.hogai.core.runner.LLM_PROVIDER_ERROR_COUNTER") as mock_provider_counter,
patch("ee.hogai.core.runner.posthoganalytics") as mock_posthog,
patch("ee.hogai.core.runner.logger") as mock_logger,
):
results = []
async for event_type, message in runner.astream(
stream_message_chunks=False, stream_first_message=False, stream_only_assistant_messages=True
):
results.append((event_type, message))
self.assertEqual(len(results), 1)
event_type, message = results[0]
self.assertEqual(event_type, AssistantEventType.MESSAGE)
self.assertIsInstance(message, FailureMessage)
self.assertEqual(
message.content,
"I'm unable to respond right now due to a temporary service issue. Please try again later.",
)
mock_graph.aupdate_state.assert_called()
# Transport counter is incremented with error_type (not provider)
mock_transport_counter.labels.assert_called_with(error_type=expected_error_type)
mock_transport_counter.labels.return_value.inc.assert_called_once()
# Provider counter is NOT incremented
mock_provider_counter.labels.assert_not_called()
# Logged as transport error, not provider error
mock_logger.exception.assert_called_once()
call_args = mock_logger.exception.call_args
self.assertEqual(call_args[0][0], "llm_transport_error")
self.assertEqual(call_args[1]["error_type"], expected_error_type)
# Captured with transport error type, no provider field
mock_posthog.capture_exception.assert_called_once()
capture_call_args = mock_posthog.capture_exception.call_args
self.assertEqual(capture_call_args[1]["properties"]["error_type"], "llm_transport_error")
self.assertNotIn("provider", capture_call_args[1]["properties"])
@parameterized.expand(
[
(
"openai_generic_api_error",
openai.APIError(
message="Unknown API error",
request=httpx.Request("POST", "https://api.openai.com/v1/chat/completions"),
body=None,
),
"openai",
),
]
)
async def test_llm_generic_api_errors_handled_gracefully(self, _name, exception, expected_provider):
"""Generic API errors (not client or transient) get a generic message."""
runner, mock_graph = self._create_mock_runner(exception)
with (
patch.object(runner, "_init_or_update_state", new_callable=AsyncMock, return_value=None),
patch.object(runner, "_lock_conversation", return_value=mock_lock_conversation()),
patch("ee.hogai.core.runner.LLM_PROVIDER_ERROR_COUNTER") as mock_counter,
patch("ee.hogai.core.runner.posthoganalytics") as mock_posthog,
patch("ee.hogai.core.runner.logger") as mock_logger,
):
results = []
async for event_type, message in runner.astream(
stream_message_chunks=False, stream_first_message=False, stream_only_assistant_messages=True
):
results.append((event_type, message))
# Verify that a FailureMessage was yielded
self.assertEqual(len(results), 1)
event_type, message = results[0]
self.assertEqual(event_type, AssistantEventType.MESSAGE)
self.assertIsInstance(message, FailureMessage)
self.assertEqual(
message.content,
"I'm unable to respond right now. Please try again later.",
)
# Verify state was reset
mock_graph.aupdate_state.assert_called()
# Verify Prometheus counter was incremented with correct provider
mock_counter.labels.assert_called_with(provider=expected_provider)
mock_counter.labels.return_value.inc.assert_called_once()
# Verify error was logged as api error
mock_logger.exception.assert_called_once()
call_args = mock_logger.exception.call_args
self.assertEqual(call_args[0][0], "llm_api_error")
self.assertEqual(call_args[1]["provider"], expected_provider)
# Verify exception was captured with correct type
mock_posthog.capture_exception.assert_called_once()
capture_call_args = mock_posthog.capture_exception.call_args
self.assertEqual(capture_call_args[1]["properties"]["error_type"], "llm_api_error")
self.assertEqual(capture_call_args[1]["properties"]["provider"], expected_provider)
class TestRunnerSubagentBehavior(BaseTest):
"""
Tests for subagent-specific behavior when use_checkpointer=False.
Subagents differ from the main agent in several ways:
- They skip interrupt handling
- They don't reset state on errors
- They don't lock the conversation
- They skip checkpoint-related state initialization
"""
def setUp(self):
super().setUp()
self.conversation = Conversation.objects.create(team=self.team, user=self.user)
def _create_runner(self, use_checkpointer: bool, graph_mock=None):
"""Create a runner with specified checkpointer setting."""
from ee.hogai.core.runner import BaseAgentRunner
mock_graph = graph_mock or MagicMock()
if not graph_mock:
mock_graph.aget_state = AsyncMock(return_value=MagicMock(values={}, next=None))
mock_graph.aupdate_state = AsyncMock()
mock_stream_processor = MagicMock()
mock_stream_processor.mark_id_as_streamed = MagicMock()
class TestRunner(BaseAgentRunner):
def get_initial_state(self):
return AssistantState(messages=[])
def get_resumed_state(self):
return PartialAssistantState(messages=[])
runner = TestRunner(
team=self.team,
conversation=self.conversation,
user=self.user,
graph_class=cast(type[BaseAssistantGraph], mock_graph),
state_type=AssistantState,
partial_state_type=PartialAssistantState,
stream_processor=mock_stream_processor,
use_checkpointer=use_checkpointer,
)
runner._graph = mock_graph
return runner, mock_graph
@parameterized.expand(
[
("with_checkpointer", True, True),
("without_checkpointer", False, False),
]
)
async def test_interrupt_handling_depends_on_checkpointer(self, _name, use_checkpointer, should_check_interrupts):
"""
When use_checkpointer=False (subagent), the runner should skip interrupt handling
and return early after streaming.
"""
async def empty_stream():
yield
return
runner, mock_graph = self._create_runner(use_checkpointer)
mock_graph.astream = MagicMock(return_value=empty_stream())
# Set up aget_state to return a state with pending next steps (interrupt)
mock_state = MagicMock()
mock_state.values = {}
mock_state.next = ["some_node"] # Indicates pending interrupt
mock_state.tasks = []
mock_graph.aget_state = AsyncMock(return_value=mock_state)
with (
patch.object(runner, "_init_or_update_state", new_callable=AsyncMock, return_value=None),
patch.object(runner, "_lock_conversation", return_value=mock_lock_conversation()),
):
results = []
async for event_type, message in runner.astream(
stream_message_chunks=False, stream_first_message=False, stream_only_assistant_messages=True
):
results.append((event_type, message))
if should_check_interrupts:
mock_graph.aget_state.assert_called()
else:
mock_graph.aget_state.assert_not_called()
@parameterized.expand(
[
("with_checkpointer", True, True),
("without_checkpointer", False, False),
]
)
async def test_state_reset_on_llm_error_depends_on_checkpointer(self, _name, use_checkpointer, should_reset_state):
"""
When use_checkpointer=False (subagent), state should NOT be reset on LLM errors.
"""
exception = anthropic.InternalServerError(
message="Internal server error",
response=httpx.Response(
status_code=500, request=httpx.Request("POST", "https://api.anthropic.com/v1/messages")
),
body=None,
)
runner, mock_graph = self._create_runner(use_checkpointer)
mock_graph.astream = MagicMock(return_value=_async_generator_that_raises(exception))
with (
patch.object(runner, "_init_or_update_state", new_callable=AsyncMock, return_value=None),
patch.object(runner, "_lock_conversation", return_value=mock_lock_conversation()),
patch("ee.hogai.core.runner.LLM_PROVIDER_ERROR_COUNTER"),
patch("ee.hogai.core.runner.posthoganalytics"),
patch("ee.hogai.core.runner.logger"),
):
results = []
async for event_type, message in runner.astream(
stream_message_chunks=False, stream_first_message=False, stream_only_assistant_messages=True
):
results.append((event_type, message))
# Both should yield a failure message
self.assertEqual(len(results), 1)
self.assertIsInstance(results[0][1], FailureMessage)
# Only checkpointer mode should reset state
if should_reset_state:
mock_graph.aupdate_state.assert_called()
else:
mock_graph.aupdate_state.assert_not_called()
@parameterized.expand(
[
("with_checkpointer", True, True),
("without_checkpointer", False, False),
]
)
async def test_state_reset_on_general_exception_depends_on_checkpointer(
self, _name, use_checkpointer, should_reset_state
):
"""
When use_checkpointer=False (subagent), state should NOT be reset on general exceptions.
"""
exception = RuntimeError("Unexpected error")
runner, mock_graph = self._create_runner(use_checkpointer)
mock_graph.astream = MagicMock(return_value=_async_generator_that_raises(exception))
# Set up state for checkpointer case
mock_state = MagicMock()
mock_state.values = {"messages": []}
mock_state.next = None
mock_graph.aget_state = AsyncMock(return_value=mock_state)
with (
patch.object(runner, "_init_or_update_state", new_callable=AsyncMock, return_value=None),
patch.object(runner, "_lock_conversation", return_value=mock_lock_conversation()),
patch("ee.hogai.core.runner.posthoganalytics"),
patch("ee.hogai.core.runner.logger"),
):
results = []
async for event_type, message in runner.astream(
stream_message_chunks=False, stream_first_message=False, stream_only_assistant_messages=True
):
results.append((event_type, message))
if should_reset_state:
mock_graph.aupdate_state.assert_called()
else:
mock_graph.aupdate_state.assert_not_called()
async def test_subagent_lock_conversation_does_not_update_status(self):
"""
When use_checkpointer=False (subagent), _lock_conversation should yield
immediately without updating conversation status.
"""
runner, _ = self._create_runner(use_checkpointer=False)
# Capture initial status
initial_status = self.conversation.status
async with runner._lock_conversation():
# Status should not have changed during the context
await self.conversation.arefresh_from_db()
self.assertEqual(self.conversation.status, initial_status)
# Status should still be unchanged after exiting
await self.conversation.arefresh_from_db()
self.assertEqual(self.conversation.status, initial_status)
async def test_main_agent_lock_conversation_updates_status(self):
"""
When use_checkpointer=True (main agent), _lock_conversation should
update conversation status to IN_PROGRESS and then to IDLE.
"""
runner, _ = self._create_runner(use_checkpointer=True)
# Ensure conversation starts as IDLE
self.conversation.status = Conversation.Status.IDLE
await self.conversation.asave()
async with runner._lock_conversation():
await self.conversation.arefresh_from_db()
self.assertEqual(self.conversation.status, Conversation.Status.IN_PROGRESS)
await self.conversation.arefresh_from_db()
self.assertEqual(self.conversation.status, Conversation.Status.IDLE)
@parameterized.expand(
[
("with_checkpointer", True, True),
("without_checkpointer", False, False),
]
)
async def test_init_or_update_state_checkpoint_lookup_depends_on_checkpointer(
self, _name, use_checkpointer, should_check_checkpoint
):
"""
When use_checkpointer=False (subagent), _init_or_update_state should skip
checkpoint state lookup and go directly to initial state.
"""
runner, mock_graph = self._create_runner(use_checkpointer)
mock_state = MagicMock()
mock_state.values = {"messages": []}
mock_state.next = None
mock_graph.aget_state = AsyncMock(return_value=mock_state)
with patch("ee.hogai.core.runner.is_cloud", return_value=False):
await runner._init_or_update_state()
if should_check_checkpoint:
mock_graph.aget_state.assert_called()
else:
mock_graph.aget_state.assert_not_called()
async def test_subagent_callback_handler_uses_parent_span_id(self):
"""
When parent_span_id is provided, SubagentCallbackHandler should be used
to nest all events under the parent span.
"""
from uuid import uuid4
from ee.hogai.core.runner import SubagentCallbackHandler
parent_span_id = uuid4()
with (
patch("ee.hogai.core.runner.is_cloud", return_value=False),
patch("ee.hogai.core.runner.posthoganalytics") as mock_posthog,
):
mock_client = MagicMock()
mock_posthog.default_client = mock_client
from ee.hogai.core.runner import BaseAgentRunner
mock_graph = MagicMock()
mock_stream_processor = MagicMock()
class TestRunner(BaseAgentRunner):
def get_initial_state(self):
return AssistantState(messages=[])
def get_resumed_state(self):
return PartialAssistantState(messages=[])
runner = TestRunner(
team=self.team,
conversation=self.conversation,
user=self.user,
graph_class=cast(type[BaseAssistantGraph], mock_graph),
state_type=AssistantState,
partial_state_type=PartialAssistantState,
stream_processor=mock_stream_processor,
use_checkpointer=False,
parent_span_id=parent_span_id,
)
# Check that SubagentCallbackHandler was used
self.assertEqual(len(runner._callback_handlers), 1)
self.assertIsInstance(runner._callback_handlers[0], SubagentCallbackHandler)
handler = cast(SubagentCallbackHandler, runner._callback_handlers[0])
self.assertEqual(handler._parent_span_id, parent_span_id)
class TestRunnerConversationTitleAction(BaseTest):
def setUp(self):
super().setUp()
self.conversation = Conversation.objects.create(team=self.team, user=self.user)
def _create_runner(self, stream_updates=None):
from ee.hogai.core.runner import BaseAgentRunner
async def _fake_astream(*args, **kwargs):
for update in stream_updates or []:
yield update
mock_graph = MagicMock()
mock_graph.astream = MagicMock(return_value=_fake_astream())
mock_graph.aget_state = AsyncMock(return_value=MagicMock(values={}, next=None))
mock_graph.aupdate_state = AsyncMock()
mock_stream_processor = MagicMock()
mock_stream_processor.mark_id_as_streamed = MagicMock()
mock_stream_processor.process = AsyncMock(return_value=None)
mock_stream_processor.process_langgraph_update = AsyncMock(return_value=None)
mock_graph_class = MagicMock()
mock_graph_instance = MagicMock()
mock_graph_instance.compile_full_graph = MagicMock(return_value=mock_graph)
mock_graph_class.return_value = mock_graph_instance
class TestRunner(BaseAgentRunner):
def get_initial_state(self):
return AssistantState(messages=[])
def get_resumed_state(self):
return PartialAssistantState(messages=[])
runner = TestRunner(
team=self.team,
conversation=self.conversation,
user=self.user,
graph_class=cast(type[BaseAssistantGraph], mock_graph_class),
state_type=AssistantState,
partial_state_type=PartialAssistantState,
stream_processor=mock_stream_processor,
)
runner._graph = mock_graph
return runner, mock_graph
async def test_process_update_sets_conversation_title_and_pending_flag(self):
runner, _ = self._create_runner()
runner._pending_conversation_update = False
# In the LangGraph custom stream, ConversationTitleAction arrives as (namespace, "custom", action)
action = ConversationTitleAction(title="My Generated Title")
raw_update = (("title_generator",), "custom", action)
result = await runner._process_update(raw_update)
self.assertIsNone(result)
self.assertEqual(runner._conversation.title, "My Generated Title")
self.assertTrue(runner._pending_conversation_update)
async def test_process_update_does_not_set_pending_flag_for_other_updates(self):
runner, _ = self._create_runner()
runner._pending_conversation_update = False
# A values update — (namespace, "values", state_dict) format from LangGraph multi-mode streaming
other_update: tuple[tuple[str, ...], str, dict] = (("some_node",), "values", {})
await runner._process_update(other_update)
self.assertFalse(runner._pending_conversation_update)
async def test_astream_yields_conversation_event_when_title_action_received(self):
title_action = ConversationTitleAction(title="Streamed Title")
# Wrap in the LangGraph custom stream tuple format
raw_update = (("title_generator",), "custom", title_action)
runner, mock_graph = self._create_runner(stream_updates=[raw_update])
with (
patch.object(runner, "_init_or_update_state", new_callable=AsyncMock, return_value=None),
patch.object(runner, "_lock_conversation", return_value=mock_lock_conversation()),
):
results = []
async for event_type, payload in runner.astream(
stream_message_chunks=False,
stream_first_message=False,
stream_only_assistant_messages=False,
):
results.append((event_type, payload))
conversation_events = [(et, p) for et, p in results if et == AssistantEventType.CONVERSATION]
self.assertEqual(len(conversation_events), 1)
_, conversation = conversation_events[0]
self.assertEqual(conversation.title, "Streamed Title")
class TestRunnerClientToolCallInterrupt(BaseTest):
"""
A ClientToolCallRequest interrupt must stream nothing and leave graph state untouched,
so the frontend handler can resume it with a payload (or a new human message can abandon
it and start fresh).
"""
def setUp(self):
super().setUp()
self.conversation = Conversation.objects.create(team=self.team, user=self.user)
def _create_runner_with_interrupt(self, interrupt_value):
from ee.hogai.core.runner import BaseAgentRunner
async def empty_stream():
return
yield
mock_graph = MagicMock()
mock_graph.astream = MagicMock(return_value=empty_stream())
mock_state = MagicMock()
mock_state.values = {}
mock_state.next = ["pending_node"]
mock_state.tasks = [MagicMock(result=None, interrupts=[MagicMock(value=interrupt_value)])]
mock_graph.aget_state = AsyncMock(return_value=mock_state)
mock_graph.aupdate_state = AsyncMock()
mock_stream_processor = MagicMock()
mock_stream_processor.mark_id_as_streamed = MagicMock()
class TestRunner(BaseAgentRunner):
def get_initial_state(self):
return AssistantState(messages=[])
def get_resumed_state(self):
return PartialAssistantState(messages=[])
runner = TestRunner(
team=self.team,
conversation=self.conversation,
user=self.user,
graph_class=cast(type[BaseAssistantGraph], mock_graph),
state_type=AssistantState,
partial_state_type=PartialAssistantState,
stream_processor=mock_stream_processor,
use_checkpointer=True,
)
runner._graph = mock_graph
return runner, mock_graph
async def _collect_message_events(self, runner):
with (
patch.object(runner, "_init_or_update_state", new_callable=AsyncMock, return_value=None),
patch.object(runner, "_lock_conversation", return_value=mock_lock_conversation()),
):
results = []
async for event_type, payload in runner.astream(
stream_message_chunks=False, stream_first_message=False, stream_only_assistant_messages=True
):
results.append((event_type, payload))
return [payload for event_type, payload in results if event_type == AssistantEventType.MESSAGE]
async def test_streams_nothing_and_skips_state_update(self):
runner, mock_graph = self._create_runner_with_interrupt(
ClientToolCallRequest(tool_name="create_form", original_tool_call_id="call_1")
)
messages = await self._collect_message_events(runner)
self.assertEqual(messages, [])
mock_graph.aupdate_state.assert_not_called()
async def test_string_interrupt_still_streams_message_and_updates_state(self):
runner, mock_graph = self._create_runner_with_interrupt("Please clarify your request")
messages = await self._collect_message_events(runner)
self.assertTrue(any(getattr(m, "content", None) == "Please clarify your request" for m in messages))
mock_graph.aupdate_state.assert_called_once()
@parameterized.expand([("approve", None), ("reject", None), ("reject", "make them monthly")])
async def test_response_targets_selected_parallel_approval(self, action: str, feedback: str | None) -> None:
executed: list[str] = []
def operation(name: str) -> Callable[[AssistantState], dict[str, object]]:
proposal_id = str(uuid4())
def run(state: AssistantState) -> dict[str, object]:
response = interrupt(
ApprovalRequest(
proposal_id=proposal_id, tool_name=name, preview=name, payload={}, original_tool_call_id=name
)
)
if response["action"] == "approve":
executed.append(name)
return {}
return run
builder = StateGraph(AssistantState)
for name in ("first", "second"):
builder.add_node(name, operation(name))
builder.add_edge(START, name)
builder.add_edge(name, END)
graph = builder.compile(checkpointer=MemorySaver())
runner, _ = self._create_runner_with_interrupt(None)
runner._graph = graph
config = runner._get_config()
config["callbacks"] = []
await graph.ainvoke(AssistantState(messages=[]), config)
snapshot = await graph.aget_state(config)
approvals = {pending.value.tool_name: pending.value for task in snapshot.tasks for pending in task.interrupts}
for approval in approvals.values():
self.conversation.approval_decisions[approval.proposal_id] = {"decision_status": "pending"}
await self.conversation.asave(update_fields=["approval_decisions"])
runner._resume_payload = {"action": action, "proposal_id": approvals["second"].proposal_id}
if feedback:
runner._resume_payload["feedback"] = feedback
command = await runner._init_or_update_state()
await graph.ainvoke(command, config)
self.assertEqual(executed, ["second"] if action == "approve" else [])
after = await graph.aget_state(config)
remaining = [pending.value for task in after.tasks if task.result is None for pending in task.interrupts]
if feedback:
await self.conversation.arefresh_from_db()
self.assertEqual(remaining, [])
self.assertEqual(
{
name: self.conversation.approval_decisions[approval.proposal_id]["decision_status"]
for name, approval in approvals.items()
},
{"first": "auto_rejected", "second": "rejected"},
)
return
self.assertEqual([request.proposal_id for request in remaining], [approvals["first"].proposal_id])
await self.conversation.arefresh_from_db()
self.assertEqual(
self.conversation.approval_decisions[approvals["first"].proposal_id]["decision_status"], "pending"
)
self.assertEqual(
self.conversation.approval_decisions[approvals["second"].proposal_id]["decision_status"],
"approved" if action == "approve" else "rejected",
)
with self.assertRaisesMessage(ValueError, "Approval does not match a pending operation"):
await runner._init_or_update_state()
runner._resume_payload = {"action": "reject", "proposal_id": approvals["first"].proposal_id}
await graph.ainvoke(await runner._init_or_update_state(), config)
self.assertFalse((await graph.aget_state(config)).next)
self.assertEqual(executed, ["second"] if action == "approve" else [])
@parameterized.expand(
[
("new_message", {"live": "auto_rejected", "stale": "auto_rejected"}),
("stale_response", {"live": "pending", "stale": "auto_rejected"}),
("stale_response_with_message", {"live": "auto_rejected", "stale": "auto_rejected"}),
("live_response", {"live": "approved", "stale": "auto_rejected"}),
]
)
async def test_unresumable_pending_approval_is_auto_rejected(self, trigger: str, expected: dict[str, str]) -> None:
live_id, stale_id = str(uuid4()), str(uuid4())
def operation(state: AssistantState) -> dict[str, object]:
interrupt(ApprovalRequest(proposal_id=live_id, tool_name="create_insight", preview="Update", payload={}))
return {}
builder = StateGraph(AssistantState)
builder.add_node("operation", operation)
builder.add_edge(START, "operation")
builder.add_edge("operation", END)
graph = builder.compile(checkpointer=MemorySaver())
runner, _ = self._create_runner_with_interrupt(None)
runner._graph = graph
config = runner._get_config()
config["callbacks"] = []
await graph.ainvoke(AssistantState(messages=[]), config)
self.conversation.approval_decisions = {
live_id: {"decision_status": "pending"},
stale_id: {"decision_status": "pending"},
}
await self.conversation.asave(update_fields=["approval_decisions"])
if trigger == "new_message":
runner._latest_message = HumanMessage(content="are you stuck?", id=str(uuid4()))
await runner._init_or_update_state()
elif trigger == "stale_response_with_message":
runner._latest_message = HumanMessage(content="are you stuck?", id=str(uuid4()))
runner._resume_payload = {"action": "reject", "proposal_id": stale_id}
self.assertIsInstance(await runner._init_or_update_state(), AssistantState)
elif trigger == "live_response":
runner._resume_payload = {"action": "approve", "proposal_id": live_id}
await runner._init_or_update_state()
else:
runner._resume_payload = {"action": "approve", "proposal_id": stale_id}
with self.assertRaisesMessage(ValueError, "Approval does not match a pending operation"):
await runner._init_or_update_state()
await self.conversation.arefresh_from_db()
self.assertEqual(
{
"live": self.conversation.approval_decisions[live_id]["decision_status"],
"stale": self.conversation.approval_decisions[stale_id]["decision_status"],
},
expected,
)