Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

23.0 KB
Raw Blame
import pytest
from posthog.test.base import BaseTest
from unittest.mock import patch

from django.test import SimpleTestCase

from langgraph.errors import GraphInterrupt
from pydantic import BaseModel

from products.access_control.backend.facade.user_access_control import UserAccessControl

from ee.hogai.core.context import set_node_path
from ee.hogai.registry import CONTEXTUAL_TOOL_NAME_TO_TOOL, _import_max_tools
from ee.hogai.tool import ApprovalResumePayload, ClientToolCallRequest, MaxTool
from ee.hogai.tool_errors import (
    MaxToolAccessDeniedError,
    MaxToolError,
    MaxToolFatalError,
    MaxToolRetryableError,
    MaxToolTransientError,
)
from ee.hogai.utils.types.base import NodePath


class DummyToolInput(BaseModel):
    input_value: str


class DummyTool(MaxTool):
    name: str = "read_taxonomy"
    description: str = "A test tool for node_path testing"
    args_schema: type[BaseModel] = DummyToolInput

    async def _arun_impl(self, input_value: str):
        return ("result", {})


class TestMaxTool(BaseTest):
    def test_format_context_prompt_injection_no_template(self):
        tool = DummyTool(team=self.team, user=self.user)
        result = tool.format_context_prompt_injection({})
        assert result is None

    def test_format_context_prompt_injection(self):
        tool = DummyTool(team=self.team, user=self.user, context_prompt_template="Test")
        result = tool.format_context_prompt_injection({})
        assert result == "Test"

    def test_format_context_prompt_injection_missing_key_defaults_to_none(self):
        tool = DummyTool(team=self.team, user=self.user, context_prompt_template="Value: {expected_key}")
        result = tool.format_context_prompt_injection({})
        assert result == "Value: None"

    def test_format_context_prompt_injection_substitutes_provided_key(self):
        tool = DummyTool(team=self.team, user=self.user, context_prompt_template="Value: {expected_key}")
        result = tool.format_context_prompt_injection({"expected_key": "hello"})
        assert result == "Value: hello"

    def test_format_context_prompt_injection_ignores_literal_braces_in_code(self):
        # Templates may contain literal `{...}` code snippets (e.g. Hog/JS examples).
        # These must not be parsed as placeholders.
        template = (
            "Inline example: `fun name(args) { ... }` and the block form\n"
            "    fun onEvent(event) {\n"
            "        // ...\n"
            "        return event\n"
            "    }\n"
            "End."
        )
        tool = DummyTool(team=self.team, user=self.user, context_prompt_template=template)
        result = tool.format_context_prompt_injection({"current_hog_code": "let x := 1"})
        assert result == template

    def test_format_context_prompt_injection_mixed_placeholders_and_literal_braces(self):
        template = "Code: { ... }\nKey: {expected_key}"
        tool = DummyTool(team=self.team, user=self.user, context_prompt_template=template)
        result = tool.format_context_prompt_injection({"expected_key": "v"})
        assert result == "Code: { ... }\nKey: v"

    def test_format_context_prompt_injection_supports_brace_escapes(self):
        # `{{key}}` should render as `{<value>}` to preserve the prior escape behavior
        # used by some templates (e.g. session-recording filters).
        template = "filters={{{current_filters}}}"
        tool = DummyTool(team=self.team, user=self.user, context_prompt_template=template)
        result = tool.format_context_prompt_injection({"current_filters": "abc"})
        assert result == "filters={abc}"


class TestMaxToolNodePath(BaseTest):
    def test_node_path_uses_context_when_not_passed(self):
        context_path = (
            NodePath(name="parent_node"),
            NodePath(name="child_node"),
        )

        with set_node_path(context_path):
            tool = DummyTool(team=self.team, user=self.user)

            result = tool.node_path

            self.assertEqual(len(result), 3)
            self.assertEqual(result[0].name, "parent_node")
            self.assertEqual(result[1].name, "child_node")
            self.assertEqual(result[2].name, "max_tool.read_taxonomy")

    def test_node_path_uses_empty_tuple_when_no_context(self):
        tool = DummyTool(team=self.team, user=self.user, node_path=None)

        result = tool.node_path

        self.assertEqual(len(result), 1)
        self.assertEqual(result[0].name, "max_tool.read_taxonomy")

    def test_node_path_uses_provided_path(self):
        provided_path = (
            NodePath(name="explicit_parent"),
            NodePath(name="explicit_child"),
        )

        tool = DummyTool(team=self.team, user=self.user, node_path=provided_path)

        result = tool.node_path

        self.assertEqual(len(result), 3)
        self.assertEqual(result[0].name, "explicit_parent")
        self.assertEqual(result[1].name, "explicit_child")
        self.assertEqual(result[2].name, "max_tool.read_taxonomy")


class TestMaxToolClientExecution(BaseTest):
    def _create_tool(self) -> DummyTool:
        return DummyTool(
            team=self.team,
            user=self.user,
            node_path=(NodePath(name="root", tool_call_id="call_123", message_id="msg_1"),),
        )

    def test_interrupts_with_client_tool_call_request(self):
        tool = self._create_tool()

        with patch("ee.hogai.tool.interrupt") as mock_interrupt:
            mock_interrupt.side_effect = GraphInterrupt()

            with self.assertRaises(GraphInterrupt):
                tool.request_client_execution()

            request = mock_interrupt.call_args.args[0]
            self.assertIsInstance(request, ClientToolCallRequest)
            self.assertEqual(request.tool_name, "read_taxonomy")
            self.assertEqual(request.original_tool_call_id, "call_123")

    def test_returns_client_result_verbatim(self):
        tool = self._create_tool()

        with patch("ee.hogai.tool.interrupt") as mock_interrupt:
            mock_interrupt.return_value = {
                "action": "client_tool_result",
                "result": {"valid": False, "error": "no rule matched"},
            }

            result = tool.request_client_execution()

            self.assertEqual(result, {"valid": False, "error": "no rule matched"})

    def test_raises_retryable_error_on_invalid_resume_payload(self):
        tool = self._create_tool()

        with patch("ee.hogai.tool.interrupt") as mock_interrupt:
            mock_interrupt.return_value = {"action": "approve", "proposal_id": "prop_1"}

            with self.assertRaises(MaxToolRetryableError):
                tool.request_client_execution()

    def test_accepts_result_addressed_to_this_tool_call(self):
        tool = self._create_tool()

        with patch("ee.hogai.tool.interrupt") as mock_interrupt:
            mock_interrupt.return_value = {
                "action": "client_tool_result",
                "tool_call_id": "call_123",
                "result": {"ok": True},
            }

            self.assertEqual(tool.request_client_execution(), {"ok": True})

    def test_raises_retryable_error_on_result_for_a_different_tool_call(self):
        tool = self._create_tool()

        with patch("ee.hogai.tool.interrupt") as mock_interrupt:
            mock_interrupt.return_value = {
                "action": "client_tool_result",
                "tool_call_id": "call_999",
                "result": {"ok": True},
            }

            with self.assertRaises(MaxToolRetryableError):
                tool.request_client_execution()


class TestMaxToolErrorHierarchy(SimpleTestCase):
    """Test the MaxToolError exception hierarchy and retry strategies."""

    def test_max_tool_error_base_has_never_retry_strategy(self):
        """Base MaxToolError should have 'never' retry strategy."""
        error = MaxToolError("Base error message")
        self.assertEqual(error.retry_strategy, "never")
        self.assertEqual(str(error), "Base error message")

    def test_max_tool_error_base_has_no_retry_hint(self):
        """Base MaxToolError should have empty retry hint."""
        error = MaxToolError("Base error message")
        self.assertEqual(error.retry_hint, "")

    def test_max_tool_fatal_error_has_never_retry_strategy(self):
        """MaxToolFatalError should have 'never' retry strategy."""
        error = MaxToolFatalError("Fatal error: missing configuration")
        self.assertEqual(error.retry_strategy, "never")
        self.assertEqual(error.retry_hint, "")
        self.assertEqual(str(error), "Fatal error: missing configuration")

    def test_max_tool_transient_error_has_once_retry_strategy(self):
        """MaxToolTransientError should have 'once' retry strategy."""
        error = MaxToolTransientError("Rate limit exceeded")
        self.assertEqual(error.retry_strategy, "once")
        self.assertEqual(error.retry_hint, " You may retry this operation once without changes.")
        self.assertEqual(str(error), "Rate limit exceeded")

    def test_max_tool_retryable_error_has_adjusted_retry_strategy(self):
        """MaxToolRetryableError should have 'adjusted' retry strategy."""
        error = MaxToolRetryableError("Invalid parameter: entity kind must be 'person' or 'session'")
        self.assertEqual(error.retry_strategy, "adjusted")
        self.assertEqual(error.retry_hint, " You may retry with adjusted inputs.")
        self.assertEqual(str(error), "Invalid parameter: entity kind must be 'person' or 'session'")

    def test_error_inheritance_hierarchy(self):
        """All error types should inherit from MaxToolError and Exception."""
        fatal = MaxToolFatalError("fatal")
        transient = MaxToolTransientError("transient")
        retryable = MaxToolRetryableError("retryable")

        # Check inheritance
        self.assertIsInstance(fatal, MaxToolError)
        self.assertIsInstance(fatal, Exception)
        self.assertIsInstance(transient, MaxToolError)
        self.assertIsInstance(transient, Exception)
        self.assertIsInstance(retryable, MaxToolError)
        self.assertIsInstance(retryable, Exception)

    def test_errors_can_be_caught_as_max_tool_error(self):
        """All error types should be catchable as MaxToolError."""
        errors = [
            MaxToolFatalError("fatal"),
            MaxToolTransientError("transient"),
            MaxToolRetryableError("retryable"),
        ]

        for error in errors:
            try:
                raise error
            except MaxToolError as e:
                self.assertIsInstance(e, MaxToolError)
                self.assertIn(e.retry_strategy, ["never", "once", "adjusted"])

    def test_error_message_preservation(self):
        """Error messages should be preserved through the exception."""
        test_message = "This is a detailed error message with context about what went wrong"

        fatal = MaxToolFatalError(test_message)
        transient = MaxToolTransientError(test_message)
        retryable = MaxToolRetryableError(test_message)

        self.assertEqual(str(fatal), test_message)
        self.assertEqual(str(transient), test_message)
        self.assertEqual(str(retryable), test_message)

    def test_retry_hint_for_all_strategies(self):
        """Each retry strategy should have appropriate retry hint."""
        never_error = MaxToolFatalError("fatal")
        once_error = MaxToolTransientError("transient")
        adjusted_error = MaxToolRetryableError("retryable")

        # Never retry should have no hint
        self.assertEqual(never_error.retry_hint, "")

        # Once retry should suggest trying once
        self.assertIn("once", once_error.retry_hint.lower())
        self.assertIn("without changes", once_error.retry_hint.lower())

        # Adjusted retry should suggest adjusting inputs
        self.assertIn("adjusted", adjusted_error.retry_hint.lower())
        self.assertIn("inputs", adjusted_error.retry_hint.lower())

    def test_to_summary_formats_error_correctly(self):
        """to_summary() should format error with class name and message."""
        error = MaxToolFatalError("Something went wrong")
        summary = error.to_summary()

        self.assertEqual(summary, "MaxToolFatalError: Something went wrong")

    def test_to_summary_truncates_long_messages(self):
        """to_summary() should truncate messages longer than max_length."""
        long_message = "a" * 600
        error = MaxToolRetryableError(long_message)
        summary = error.to_summary(max_length=500)

        self.assertEqual(len(summary), 524)  # "MaxToolRetryableError: " (24) + 500 + "…" (1) = 525
        self.assertTrue(summary.startswith("MaxToolRetryableError: " + "a" * 500))
        self.assertTrue(summary.endswith("…"))

    def test_to_summary_respects_custom_max_length(self):
        """to_summary() should respect custom max_length parameter."""
        error = MaxToolTransientError("This is a medium length error message")
        summary = error.to_summary(max_length=20)

        self.assertTrue(summary.startswith("MaxToolTransientError: This is a medium len"))
        self.assertTrue(summary.endswith("…"))
        self.assertEqual(len(summary), 44)  # "MaxToolTransientError: " (23) + 20 + "…" (1) = 44

    def test_to_summary_strips_whitespace(self):
        """to_summary() should strip leading/trailing whitespace from messages."""
        error = MaxToolFatalError("  \n  Error with whitespace  \n  ")
        summary = error.to_summary()

        self.assertEqual(summary, "MaxToolFatalError: Error with whitespace")


class TestMaxToolAccessDeniedError(SimpleTestCase):
    """Test the MaxToolAccessDeniedError exception."""

    def test_access_denied_error_formats_correctly(self):
        """Access denied error should format correctly."""
        error = MaxToolAccessDeniedError("feature_flag", "editor")
        self.assertIn("editor", str(error))
        self.assertIn("feature_flag", str(error))
        self.assertIn("contact their project admin", str(error))
        self.assertEqual(error.resource, "feature_flag")
        self.assertEqual(error.required_level, "editor")

    def test_access_denied_error_with_custom_action(self):
        """Access denied error with custom action should format correctly."""
        error = MaxToolAccessDeniedError("dashboard", "editor", action="create")
        self.assertIn("create", str(error))
        self.assertIn("dashboard", str(error))

    def test_access_denied_is_fatal_error(self):
        """Access denied error should be a fatal error with 'never' retry strategy."""
        error = MaxToolAccessDeniedError("feature_flag", "editor")
        self.assertIsInstance(error, MaxToolFatalError)
        self.assertEqual(error.retry_strategy, "never")


class ToolWithAccessControlTool(MaxTool):
    """Test tool with access control requirements."""

    name: str = "create_feature_flag"
    description: str = "A test tool with access control"
    args_schema: type[BaseModel] = DummyToolInput

    def get_required_resource_access(self):
        return [("feature_flag", "editor")]

    async def _arun_impl(self, input_value: str):
        return ("result", {})


class ToolWithNoAccessControlTool(MaxTool):
    """Test tool without access control requirements."""

    name: str = "read_taxonomy"
    description: str = "A test tool without access control"
    args_schema: type[BaseModel] = DummyToolInput

    def get_required_resource_access(self):
        return []

    async def _arun_impl(self, input_value: str):
        return ("result", {})


class TestMaxToolAccessControl(BaseTest):
    """Test the MaxTool resource-level access control functionality."""

    def test_access_control_property_returns_user_access_control(self):
        """access_control property should return UserAccessControl instance."""
        tool = DummyTool(team=self.team, user=self.user)
        access_control = tool.user_access_control
        self.assertIsInstance(access_control, UserAccessControl)

    def test_get_required_resource_access_default_returns_empty(self):
        """Default get_required_resource_access should return empty list."""
        tool = DummyTool(team=self.team, user=self.user)
        self.assertEqual(tool.get_required_resource_access(), [])

    def test_get_required_resource_access_can_be_overridden(self):
        """get_required_resource_access can be overridden to return requirements."""
        tool = ToolWithAccessControlTool(team=self.team, user=self.user)
        requirements = tool.get_required_resource_access()
        self.assertEqual(requirements, [("feature_flag", "editor")])

    def test_check_access_control_passes_when_user_has_access(self):
        """_check_access_control should pass when user has required access."""
        tool = ToolWithAccessControlTool(team=self.team, user=self.user)

        with patch.object(tool.user_access_control, "check_access_level_for_resource", return_value=True):
            # Should not raise
            tool._check_resource_access()

    def test_check_access_control_raises_when_user_lacks_access(self):
        """_check_access_control should raise MaxToolAccessDeniedError when user lacks access."""
        tool = ToolWithAccessControlTool(team=self.team, user=self.user)

        with patch.object(tool.user_access_control, "check_access_level_for_resource", return_value=False):
            with self.assertRaises(MaxToolAccessDeniedError) as ctx:
                tool._check_resource_access()

            self.assertEqual(ctx.exception.resource, "feature_flag")
            self.assertEqual(ctx.exception.required_level, "editor")

    def test_check_access_control_skips_when_no_requirements(self):
        """_check_access_control should skip when get_required_resource_access returns empty list."""
        tool = ToolWithNoAccessControlTool(team=self.team, user=self.user)

        # Should not call check_access_level_for_resource at all
        with patch.object(tool.user_access_control, "check_access_level_for_resource") as mock:
            tool._check_resource_access()
            mock.assert_not_called()


class TestToolAccessControlDeclarations(BaseTest):
    """
    Test that all tools declare their access control requirements.

    New tools MUST either:
    1. Implement get_required_resource_access() returning non-empty list, OR
    2. Be explicitly added to TOOLS_WITHOUT_ACCESS_CONTROL with a reason
    """

    # Tools that are explicitly exempt from access control.
    # Add a comment explaining WHY the tool doesn't need access control.
    TOOLS_WITHOUT_ACCESS_CONTROL: set[str] = {
        # Tools that don't view or modify protected resources
        "search",
        "read_taxonomy",
        "create_form",
        "todo_write",
        "switch_mode",
        "session_summarization",
        "manage_memories",  # Manages per-team/user memories, no protected resources
        # Tools with dynamic/conditional access checks inside _arun_impl
        "read_data",
        "list_data",  # Lists entities with pagination, no protected resources modified
        "create_notebook",  # No protected resources modified
        "finalize_plan",
        # TODO: Add access control to these tools
        "task",
        "create_task",
        "run_task",
        "get_task_run",
        "get_task_run_logs",
        "list_tasks",
        "list_task_runs",
        "list_repositories",
        "create_message_template",
        "create_hog_function_inputs",
        "create_hog_transformation_function",
        "create_hog_function_filters",
        "execute_sql",
        "generate_hogql_query",
        "fix_hogql_query",
        "call_mcp_server",  # Scoped to user's own MCP installations (team + user filtered) but no protected resources modified
        "diagnose_proxy",  # Explicit OrganizationMembership.Level >= ADMIN check inside _arun_impl; resource-level RBAC doesn't recognize membership level so we can't use get_required_resource_access here
    }

    def test_all_tools_have_access_control_or_are_exempt(self):
        """All tools must declare access control or be explicitly exempt."""
        _import_max_tools()

        missing_access_control = []
        for tool_name, tool_class in CONTEXTUAL_TOOL_NAME_TO_TOOL.items():
            if tool_name.value in self.TOOLS_WITHOUT_ACCESS_CONTROL:
                continue

            # Check that get_required_resource_access() is implemented
            tool_instance = tool_class(team=self.team, user=self.user, description="Test description")
            required_access = tool_instance.get_required_resource_access()

            if not required_access:
                missing_access_control.append(tool_name)

        if missing_access_control:
            self.fail(
                f"Tools without access control declaration: {missing_access_control}. "
                f"Either add get_required_resource_access() or add to TOOLS_WITHOUT_ACCESS_CONTROL with a reason."
            )


class _ApprovalArgs(BaseModel):
    count: int


class TestDangerousOperationBindsApprovedArguments(BaseTest):
    """An approving user may edit the arguments; the tool must run what they approved, not what was asked."""

    class _SpendingTool(MaxTool):
        name: str = "create_feature_flag"
        description: str = "test"
        thinking_message: str = "test"
        args_schema: type[BaseModel] = _ApprovalArgs

        def get_required_resource_access(self):
            # Every registered tool must declare this, and subclassing registers under the shared name.
            return [("feature_flag", "editor")]

        async def is_dangerous_operation(self, **kwargs) -> bool:
            return True

        async def _arun_impl(self, count: int) -> tuple[str, dict]:
            return f"ran with {count}", {"count": count}

    @pytest.mark.asyncio
    async def test_edited_arguments_reach_the_implementation(self):
        # The approval payload used to be merged into a local copy of kwargs, so an edit at the prompt
        # was discarded and the original operation ran. On a tool that spends money, that charges for
        # something the user did not approve.
        tool = self._SpendingTool(team=self.team, user=self.user)
        approved = ApprovalResumePayload(action="approve", payload={"count": 5}, proposal_id="p1")

        with patch("ee.hogai.tool.interrupt", return_value=approved.model_dump()):
            content, artifact = await tool._arun_with_context(count=200)

        assert artifact["count"] == 5
        assert "5" in content

    def test_parallel_resumption_keeps_approval_identity(self) -> None:
        proposals = []
        for call_id in ("first", "second", "first"):
            tool = self._SpendingTool(
                team=self.team,
                user=self.user,
                node_path=(NodePath(name="root", tool_call_id=call_id),),
                config={"configurable": {"thread_id": "approval-test"}},
            )
            with patch("ee.hogai.tool.interrupt", side_effect=GraphInterrupt()) as mocked_interrupt:
                with self.assertRaises(GraphInterrupt):
                    tool._handle_dangerous_operation({"count": 5}, preview="Change count")
                proposals.append(mocked_interrupt.call_args.args[0].proposal_id)
        self.assertEqual(proposals[0], proposals[2])
        self.assertNotEqual(proposals[0], proposals[1])