:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import pytest
from posthog.test.base import BaseTest
from unittest.mock import patch
from django.test import SimpleTestCase
from langgraph.errors import GraphInterrupt
from pydantic import BaseModel
from products.access_control.backend.facade.user_access_control import UserAccessControl
from ee.hogai.core.context import set_node_path
from ee.hogai.registry import CONTEXTUAL_TOOL_NAME_TO_TOOL, _import_max_tools
from ee.hogai.tool import ApprovalResumePayload, ClientToolCallRequest, MaxTool
from ee.hogai.tool_errors import (
MaxToolAccessDeniedError,
MaxToolError,
MaxToolFatalError,
MaxToolRetryableError,
MaxToolTransientError,
)
from ee.hogai.utils.types.base import NodePath
class DummyToolInput(BaseModel):
input_value: str
class DummyTool(MaxTool):
name: str = "read_taxonomy"
description: str = "A test tool for node_path testing"
args_schema: type[BaseModel] = DummyToolInput
async def _arun_impl(self, input_value: str):
return ("result", {})
class TestMaxTool(BaseTest):
def test_format_context_prompt_injection_no_template(self):
tool = DummyTool(team=self.team, user=self.user)
result = tool.format_context_prompt_injection({})
assert result is None
def test_format_context_prompt_injection(self):
tool = DummyTool(team=self.team, user=self.user, context_prompt_template="Test")
result = tool.format_context_prompt_injection({})
assert result == "Test"
def test_format_context_prompt_injection_missing_key_defaults_to_none(self):
tool = DummyTool(team=self.team, user=self.user, context_prompt_template="Value: {expected_key}")
result = tool.format_context_prompt_injection({})
assert result == "Value: None"
def test_format_context_prompt_injection_substitutes_provided_key(self):
tool = DummyTool(team=self.team, user=self.user, context_prompt_template="Value: {expected_key}")
result = tool.format_context_prompt_injection({"expected_key": "hello"})
assert result == "Value: hello"
def test_format_context_prompt_injection_ignores_literal_braces_in_code(self):
# Templates may contain literal `{...}` code snippets (e.g. Hog/JS examples).
# These must not be parsed as placeholders.
template = (
"Inline example: `fun name(args) { ... }` and the block form\n"
" fun onEvent(event) {\n"
" // ...\n"
" return event\n"
" }\n"
"End."
)
tool = DummyTool(team=self.team, user=self.user, context_prompt_template=template)
result = tool.format_context_prompt_injection({"current_hog_code": "let x := 1"})
assert result == template
def test_format_context_prompt_injection_mixed_placeholders_and_literal_braces(self):
template = "Code: { ... }\nKey: {expected_key}"
tool = DummyTool(team=self.team, user=self.user, context_prompt_template=template)
result = tool.format_context_prompt_injection({"expected_key": "v"})
assert result == "Code: { ... }\nKey: v"
def test_format_context_prompt_injection_supports_brace_escapes(self):
# `{{key}}` should render as `{<value>}` to preserve the prior escape behavior
# used by some templates (e.g. session-recording filters).
template = "filters={{{current_filters}}}"
tool = DummyTool(team=self.team, user=self.user, context_prompt_template=template)
result = tool.format_context_prompt_injection({"current_filters": "abc"})
assert result == "filters={abc}"
class TestMaxToolNodePath(BaseTest):
def test_node_path_uses_context_when_not_passed(self):
context_path = (
NodePath(name="parent_node"),
NodePath(name="child_node"),
)
with set_node_path(context_path):
tool = DummyTool(team=self.team, user=self.user)
result = tool.node_path
self.assertEqual(len(result), 3)
self.assertEqual(result[0].name, "parent_node")
self.assertEqual(result[1].name, "child_node")
self.assertEqual(result[2].name, "max_tool.read_taxonomy")
def test_node_path_uses_empty_tuple_when_no_context(self):
tool = DummyTool(team=self.team, user=self.user, node_path=None)
result = tool.node_path
self.assertEqual(len(result), 1)
self.assertEqual(result[0].name, "max_tool.read_taxonomy")
def test_node_path_uses_provided_path(self):
provided_path = (
NodePath(name="explicit_parent"),
NodePath(name="explicit_child"),
)
tool = DummyTool(team=self.team, user=self.user, node_path=provided_path)
result = tool.node_path
self.assertEqual(len(result), 3)
self.assertEqual(result[0].name, "explicit_parent")
self.assertEqual(result[1].name, "explicit_child")
self.assertEqual(result[2].name, "max_tool.read_taxonomy")
class TestMaxToolClientExecution(BaseTest):
def _create_tool(self) -> DummyTool:
return DummyTool(
team=self.team,
user=self.user,
node_path=(NodePath(name="root", tool_call_id="call_123", message_id="msg_1"),),
)
def test_interrupts_with_client_tool_call_request(self):
tool = self._create_tool()
with patch("ee.hogai.tool.interrupt") as mock_interrupt:
mock_interrupt.side_effect = GraphInterrupt()
with self.assertRaises(GraphInterrupt):
tool.request_client_execution()
request = mock_interrupt.call_args.args[0]
self.assertIsInstance(request, ClientToolCallRequest)
self.assertEqual(request.tool_name, "read_taxonomy")
self.assertEqual(request.original_tool_call_id, "call_123")
def test_returns_client_result_verbatim(self):
tool = self._create_tool()
with patch("ee.hogai.tool.interrupt") as mock_interrupt:
mock_interrupt.return_value = {
"action": "client_tool_result",
"result": {"valid": False, "error": "no rule matched"},
}
result = tool.request_client_execution()
self.assertEqual(result, {"valid": False, "error": "no rule matched"})
def test_raises_retryable_error_on_invalid_resume_payload(self):
tool = self._create_tool()
with patch("ee.hogai.tool.interrupt") as mock_interrupt:
mock_interrupt.return_value = {"action": "approve", "proposal_id": "prop_1"}
with self.assertRaises(MaxToolRetryableError):
tool.request_client_execution()
def test_accepts_result_addressed_to_this_tool_call(self):
tool = self._create_tool()
with patch("ee.hogai.tool.interrupt") as mock_interrupt:
mock_interrupt.return_value = {
"action": "client_tool_result",
"tool_call_id": "call_123",
"result": {"ok": True},
}
self.assertEqual(tool.request_client_execution(), {"ok": True})
def test_raises_retryable_error_on_result_for_a_different_tool_call(self):
tool = self._create_tool()
with patch("ee.hogai.tool.interrupt") as mock_interrupt:
mock_interrupt.return_value = {
"action": "client_tool_result",
"tool_call_id": "call_999",
"result": {"ok": True},
}
with self.assertRaises(MaxToolRetryableError):
tool.request_client_execution()
class TestMaxToolErrorHierarchy(SimpleTestCase):
"""Test the MaxToolError exception hierarchy and retry strategies."""
def test_max_tool_error_base_has_never_retry_strategy(self):
"""Base MaxToolError should have 'never' retry strategy."""
error = MaxToolError("Base error message")
self.assertEqual(error.retry_strategy, "never")
self.assertEqual(str(error), "Base error message")
def test_max_tool_error_base_has_no_retry_hint(self):
"""Base MaxToolError should have empty retry hint."""
error = MaxToolError("Base error message")
self.assertEqual(error.retry_hint, "")
def test_max_tool_fatal_error_has_never_retry_strategy(self):
"""MaxToolFatalError should have 'never' retry strategy."""
error = MaxToolFatalError("Fatal error: missing configuration")
self.assertEqual(error.retry_strategy, "never")
self.assertEqual(error.retry_hint, "")
self.assertEqual(str(error), "Fatal error: missing configuration")
def test_max_tool_transient_error_has_once_retry_strategy(self):
"""MaxToolTransientError should have 'once' retry strategy."""
error = MaxToolTransientError("Rate limit exceeded")
self.assertEqual(error.retry_strategy, "once")
self.assertEqual(error.retry_hint, " You may retry this operation once without changes.")
self.assertEqual(str(error), "Rate limit exceeded")
def test_max_tool_retryable_error_has_adjusted_retry_strategy(self):
"""MaxToolRetryableError should have 'adjusted' retry strategy."""
error = MaxToolRetryableError("Invalid parameter: entity kind must be 'person' or 'session'")
self.assertEqual(error.retry_strategy, "adjusted")
self.assertEqual(error.retry_hint, " You may retry with adjusted inputs.")
self.assertEqual(str(error), "Invalid parameter: entity kind must be 'person' or 'session'")
def test_error_inheritance_hierarchy(self):
"""All error types should inherit from MaxToolError and Exception."""
fatal = MaxToolFatalError("fatal")
transient = MaxToolTransientError("transient")
retryable = MaxToolRetryableError("retryable")
# Check inheritance
self.assertIsInstance(fatal, MaxToolError)
self.assertIsInstance(fatal, Exception)
self.assertIsInstance(transient, MaxToolError)
self.assertIsInstance(transient, Exception)
self.assertIsInstance(retryable, MaxToolError)
self.assertIsInstance(retryable, Exception)
def test_errors_can_be_caught_as_max_tool_error(self):
"""All error types should be catchable as MaxToolError."""
errors = [
MaxToolFatalError("fatal"),
MaxToolTransientError("transient"),
MaxToolRetryableError("retryable"),
]
for error in errors:
try:
raise error
except MaxToolError as e:
self.assertIsInstance(e, MaxToolError)
self.assertIn(e.retry_strategy, ["never", "once", "adjusted"])
def test_error_message_preservation(self):
"""Error messages should be preserved through the exception."""
test_message = "This is a detailed error message with context about what went wrong"
fatal = MaxToolFatalError(test_message)
transient = MaxToolTransientError(test_message)
retryable = MaxToolRetryableError(test_message)
self.assertEqual(str(fatal), test_message)
self.assertEqual(str(transient), test_message)
self.assertEqual(str(retryable), test_message)
def test_retry_hint_for_all_strategies(self):
"""Each retry strategy should have appropriate retry hint."""
never_error = MaxToolFatalError("fatal")
once_error = MaxToolTransientError("transient")
adjusted_error = MaxToolRetryableError("retryable")
# Never retry should have no hint
self.assertEqual(never_error.retry_hint, "")
# Once retry should suggest trying once
self.assertIn("once", once_error.retry_hint.lower())
self.assertIn("without changes", once_error.retry_hint.lower())
# Adjusted retry should suggest adjusting inputs
self.assertIn("adjusted", adjusted_error.retry_hint.lower())
self.assertIn("inputs", adjusted_error.retry_hint.lower())
def test_to_summary_formats_error_correctly(self):
"""to_summary() should format error with class name and message."""
error = MaxToolFatalError("Something went wrong")
summary = error.to_summary()
self.assertEqual(summary, "MaxToolFatalError: Something went wrong")
def test_to_summary_truncates_long_messages(self):
"""to_summary() should truncate messages longer than max_length."""
long_message = "a" * 600
error = MaxToolRetryableError(long_message)
summary = error.to_summary(max_length=500)
self.assertEqual(len(summary), 524) # "MaxToolRetryableError: " (24) + 500 + "…" (1) = 525
self.assertTrue(summary.startswith("MaxToolRetryableError: " + "a" * 500))
self.assertTrue(summary.endswith("…"))
def test_to_summary_respects_custom_max_length(self):
"""to_summary() should respect custom max_length parameter."""
error = MaxToolTransientError("This is a medium length error message")
summary = error.to_summary(max_length=20)
self.assertTrue(summary.startswith("MaxToolTransientError: This is a medium len"))
self.assertTrue(summary.endswith("…"))
self.assertEqual(len(summary), 44) # "MaxToolTransientError: " (23) + 20 + "…" (1) = 44
def test_to_summary_strips_whitespace(self):
"""to_summary() should strip leading/trailing whitespace from messages."""
error = MaxToolFatalError(" \n Error with whitespace \n ")
summary = error.to_summary()
self.assertEqual(summary, "MaxToolFatalError: Error with whitespace")
class TestMaxToolAccessDeniedError(SimpleTestCase):
"""Test the MaxToolAccessDeniedError exception."""
def test_access_denied_error_formats_correctly(self):
"""Access denied error should format correctly."""
error = MaxToolAccessDeniedError("feature_flag", "editor")
self.assertIn("editor", str(error))
self.assertIn("feature_flag", str(error))
self.assertIn("contact their project admin", str(error))
self.assertEqual(error.resource, "feature_flag")
self.assertEqual(error.required_level, "editor")
def test_access_denied_error_with_custom_action(self):
"""Access denied error with custom action should format correctly."""
error = MaxToolAccessDeniedError("dashboard", "editor", action="create")
self.assertIn("create", str(error))
self.assertIn("dashboard", str(error))
def test_access_denied_is_fatal_error(self):
"""Access denied error should be a fatal error with 'never' retry strategy."""
error = MaxToolAccessDeniedError("feature_flag", "editor")
self.assertIsInstance(error, MaxToolFatalError)
self.assertEqual(error.retry_strategy, "never")
class ToolWithAccessControlTool(MaxTool):
"""Test tool with access control requirements."""
name: str = "create_feature_flag"
description: str = "A test tool with access control"
args_schema: type[BaseModel] = DummyToolInput
def get_required_resource_access(self):
return [("feature_flag", "editor")]
async def _arun_impl(self, input_value: str):
return ("result", {})
class ToolWithNoAccessControlTool(MaxTool):
"""Test tool without access control requirements."""
name: str = "read_taxonomy"
description: str = "A test tool without access control"
args_schema: type[BaseModel] = DummyToolInput
def get_required_resource_access(self):
return []
async def _arun_impl(self, input_value: str):
return ("result", {})
class TestMaxToolAccessControl(BaseTest):
"""Test the MaxTool resource-level access control functionality."""
def test_access_control_property_returns_user_access_control(self):
"""access_control property should return UserAccessControl instance."""
tool = DummyTool(team=self.team, user=self.user)
access_control = tool.user_access_control
self.assertIsInstance(access_control, UserAccessControl)
def test_get_required_resource_access_default_returns_empty(self):
"""Default get_required_resource_access should return empty list."""
tool = DummyTool(team=self.team, user=self.user)
self.assertEqual(tool.get_required_resource_access(), [])
def test_get_required_resource_access_can_be_overridden(self):
"""get_required_resource_access can be overridden to return requirements."""
tool = ToolWithAccessControlTool(team=self.team, user=self.user)
requirements = tool.get_required_resource_access()
self.assertEqual(requirements, [("feature_flag", "editor")])
def test_check_access_control_passes_when_user_has_access(self):
"""_check_access_control should pass when user has required access."""
tool = ToolWithAccessControlTool(team=self.team, user=self.user)
with patch.object(tool.user_access_control, "check_access_level_for_resource", return_value=True):
# Should not raise
tool._check_resource_access()
def test_check_access_control_raises_when_user_lacks_access(self):
"""_check_access_control should raise MaxToolAccessDeniedError when user lacks access."""
tool = ToolWithAccessControlTool(team=self.team, user=self.user)
with patch.object(tool.user_access_control, "check_access_level_for_resource", return_value=False):
with self.assertRaises(MaxToolAccessDeniedError) as ctx:
tool._check_resource_access()
self.assertEqual(ctx.exception.resource, "feature_flag")
self.assertEqual(ctx.exception.required_level, "editor")
def test_check_access_control_skips_when_no_requirements(self):
"""_check_access_control should skip when get_required_resource_access returns empty list."""
tool = ToolWithNoAccessControlTool(team=self.team, user=self.user)
# Should not call check_access_level_for_resource at all
with patch.object(tool.user_access_control, "check_access_level_for_resource") as mock:
tool._check_resource_access()
mock.assert_not_called()
class TestToolAccessControlDeclarations(BaseTest):
"""
Test that all tools declare their access control requirements.
New tools MUST either:
1. Implement get_required_resource_access() returning non-empty list, OR
2. Be explicitly added to TOOLS_WITHOUT_ACCESS_CONTROL with a reason
"""
# Tools that are explicitly exempt from access control.
# Add a comment explaining WHY the tool doesn't need access control.
TOOLS_WITHOUT_ACCESS_CONTROL: set[str] = {
# Tools that don't view or modify protected resources
"search",
"read_taxonomy",
"create_form",
"todo_write",
"switch_mode",
"session_summarization",
"manage_memories", # Manages per-team/user memories, no protected resources
# Tools with dynamic/conditional access checks inside _arun_impl
"read_data",
"list_data", # Lists entities with pagination, no protected resources modified
"create_notebook", # No protected resources modified
"finalize_plan",
# TODO: Add access control to these tools
"task",
"create_task",
"run_task",
"get_task_run",
"get_task_run_logs",
"list_tasks",
"list_task_runs",
"list_repositories",
"create_message_template",
"create_hog_function_inputs",
"create_hog_transformation_function",
"create_hog_function_filters",
"execute_sql",
"generate_hogql_query",
"fix_hogql_query",
"call_mcp_server", # Scoped to user's own MCP installations (team + user filtered) but no protected resources modified
"diagnose_proxy", # Explicit OrganizationMembership.Level >= ADMIN check inside _arun_impl; resource-level RBAC doesn't recognize membership level so we can't use get_required_resource_access here
}
def test_all_tools_have_access_control_or_are_exempt(self):
"""All tools must declare access control or be explicitly exempt."""
_import_max_tools()
missing_access_control = []
for tool_name, tool_class in CONTEXTUAL_TOOL_NAME_TO_TOOL.items():
if tool_name.value in self.TOOLS_WITHOUT_ACCESS_CONTROL:
continue
# Check that get_required_resource_access() is implemented
tool_instance = tool_class(team=self.team, user=self.user, description="Test description")
required_access = tool_instance.get_required_resource_access()
if not required_access:
missing_access_control.append(tool_name)
if missing_access_control:
self.fail(
f"Tools without access control declaration: {missing_access_control}. "
f"Either add get_required_resource_access() or add to TOOLS_WITHOUT_ACCESS_CONTROL with a reason."
)
class _ApprovalArgs(BaseModel):
count: int
class TestDangerousOperationBindsApprovedArguments(BaseTest):
"""An approving user may edit the arguments; the tool must run what they approved, not what was asked."""
class _SpendingTool(MaxTool):
name: str = "create_feature_flag"
description: str = "test"
thinking_message: str = "test"
args_schema: type[BaseModel] = _ApprovalArgs
def get_required_resource_access(self):
# Every registered tool must declare this, and subclassing registers under the shared name.
return [("feature_flag", "editor")]
async def is_dangerous_operation(self, **kwargs) -> bool:
return True
async def _arun_impl(self, count: int) -> tuple[str, dict]:
return f"ran with {count}", {"count": count}
@pytest.mark.asyncio
async def test_edited_arguments_reach_the_implementation(self):
# The approval payload used to be merged into a local copy of kwargs, so an edit at the prompt
# was discarded and the original operation ran. On a tool that spends money, that charges for
# something the user did not approve.
tool = self._SpendingTool(team=self.team, user=self.user)
approved = ApprovalResumePayload(action="approve", payload={"count": 5}, proposal_id="p1")
with patch("ee.hogai.tool.interrupt", return_value=approved.model_dump()):
content, artifact = await tool._arun_with_context(count=200)
assert artifact["count"] == 5
assert "5" in content
def test_parallel_resumption_keeps_approval_identity(self) -> None:
proposals = []
for call_id in ("first", "second", "first"):
tool = self._SpendingTool(
team=self.team,
user=self.user,
node_path=(NodePath(name="root", tool_call_id=call_id),),
config={"configurable": {"thread_id": "approval-test"}},
)
with patch("ee.hogai.tool.interrupt", side_effect=GraphInterrupt()) as mocked_interrupt:
with self.assertRaises(GraphInterrupt):
tool._handle_dangerous_operation({"count": 5}, preview="Change count")
proposals.append(mocked_interrupt.call_args.args[0].proposal_id)
self.assertEqual(proposals[0], proposals[2])
self.assertNotEqual(proposals[0], proposals[1])