Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

11.2 KB
Raw Blame
from posthog.test.base import BaseTest
from unittest.mock import Mock, patch

from asgiref.sync import async_to_sync
from langchain_core.runnables import RunnableConfig
from parameterized import parameterized

from posthog.schema import HumanMessage, MaxNotebookContext, MaxUIContext

from posthog.constants import AvailableFeature
from posthog.models import OrganizationMembership
from posthog.models.sharing_configuration import SharingConfiguration

from products.access_control.backend.models.access_control import AccessControl
from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery
from products.notebooks.backend.models import Notebook
from products.posthog_ai.backend.models.assistant import Conversation

from ee.hogai.context import AssistantContextManager
from ee.hogai.context.notebook.prompts import LEGACY_CELL_GUIDANCE, SQL_V2_CELL_GUIDANCE
from ee.hogai.tools.create_notebook.tool import CreateNotebookTool
from ee.hogai.utils.types.base import AssistantState, NodePath


class TestCreateNotebookTool(BaseTest):
    def setUp(self):
        super().setUp()
        self.conversation = Conversation.objects.create(user=self.user, team=self.team)
        self.config = RunnableConfig(configurable={"thread_id": str(self.conversation.id)})
        self.context_manager = AssistantContextManager(self.team, self.user, self.config)
        self.tool_call_id = "test_tool_call_id"
        self.node_path = (NodePath(name="test_node", tool_call_id=self.tool_call_id, message_id="test"),)
        self.tool = CreateNotebookTool(
            team=self.team,
            user=self.user,
            state=AssistantState(messages=[]),
            config=self.config,
            context_manager=self.context_manager,
            node_path=self.node_path,
        )

    def test_description_includes_object_widget_views(self) -> None:
        assert "FeatureFlag" in self.tool.description
        assert "summary: Show the flag status" in self.tool.description
        assert "editor: Edit the flag status" in self.tool.description
        assert "Cohort" in self.tool.description

    @parameterized.expand(
        [
            ("sql_v2", True, SQL_V2_CELL_GUIDANCE, LEGACY_CELL_GUIDANCE),
            ("legacy_sql", False, LEGACY_CELL_GUIDANCE, SQL_V2_CELL_GUIDANCE),
        ]
    )
    def test_description_offers_only_runnable_cell_tags(
        self, _name: str, sql_v2_enabled: bool, expected: str, unexpected: str
    ) -> None:
        with (
            patch("products.notebooks.backend.facade.api.is_sql_v2_enabled", return_value=sql_v2_enabled),
        ):
            tool = async_to_sync(CreateNotebookTool.create_tool_class)(team=self.team, user=self.user)

        assert expected in tool.description
        assert unexpected not in tool.description
        assert '<Widget title="' in tool.description
        assert "click Generate widget" in tool.description

    def test_returns_error_when_both_content_and_draft_content_provided(self):
        result, artifact = async_to_sync(self.tool._arun_impl)(
            title="Test Notebook",
            content="# Hello",
            draft_content="# Draft Hello",
        )

        assert "Cannot provide both" in result
        assert "Use exactly one" in result
        assert artifact is None

    def test_returns_error_when_neither_content_nor_draft_content_provided(self):
        result, artifact = async_to_sync(self.tool._arun_impl)(
            title="Test Notebook",
            content=None,
            draft_content=None,
        )

        assert "Either 'content' or 'draft_content' must be provided" in result
        assert artifact is None

    def test_creates_notebook_with_content(self):
        result, artifact = async_to_sync(self.tool._arun_impl)(
            title="Test Notebook",
            content="# Hello World",
        )

        assert result == ""
        assert artifact is not None
        assert len(artifact.messages) == 2

    def test_creates_draft_notebook_without_streaming(self):
        result, artifact = async_to_sync(self.tool._arun_impl)(
            title="Test Draft",
            draft_content="# Draft Content",
        )

        assert "artifact_id" in result
        assert artifact is None

    def test_update_failure_falls_back_to_create(self):
        result, artifact = async_to_sync(self.tool._arun_impl)(
            title="Test Notebook",
            content="# Hello",
            artifact_id="nonexistent_artifact_id",
        )

        assert result == ""
        assert artifact is not None
        assert len(artifact.messages) == 2
        tool_call_message = artifact.messages[1]
        assert "Failed to update" in tool_call_message.content
        assert "new artifact has been created" in tool_call_message.content

    def test_successful_update_returns_update_message(self):
        create_result, create_artifact = async_to_sync(self.tool._arun_impl)(
            title="Original Notebook",
            content="# Original",
        )
        original_artifact_id = create_artifact.messages[1].content.split("artifact_id: ")[1].split(".")[0]

        update_result, update_artifact = async_to_sync(self.tool._arun_impl)(
            title="Updated Notebook",
            content="# Updated Content",
            artifact_id=original_artifact_id,
        )

        assert update_result == ""
        assert update_artifact is not None
        tool_call_message = update_artifact.messages[1]
        assert "has been updated" in tool_call_message.content
        assert "Failed" not in tool_call_message.content

    def test_transient_notebook_message_mentions_transient(self):
        result, artifact = async_to_sync(self.tool._arun_impl)(
            title="Test Notebook",
            content="# Hello World",
        )

        assert artifact is not None
        tool_call_message = artifact.messages[1]
        assert "transient" in tool_call_message.content
        assert Notebook.objects.filter(team=self.team).count() == 0

    def test_save_to_notebook_creates_real_notebook(self):
        result, artifact = async_to_sync(self.tool._arun_impl)(
            title="Saved Notebook",
            content="# Hello World",
            save_to_notebook=True,
        )

        assert artifact is not None
        tool_call_message = artifact.messages[1]
        assert "saved" in tool_call_message.content.lower()
        assert "/notebooks/" in tool_call_message.content

        notebooks = Notebook.objects.filter(team=self.team)
        assert notebooks.count() == 1
        notebook = notebooks.first()
        assert notebook.title == "Saved Notebook"
        assert notebook.content is not None
        assert notebook.content["type"] == "doc"

    @parameterized.expand([(False,), (True,)])
    def test_inline_artifact_does_not_save_a_second_notebook(self, save_to_notebook: bool) -> None:
        notebook = Notebook.objects.create(team=self.team, created_by=self.user, title="Chart notes")
        self.tool._state = AssistantState(
            messages=[
                HumanMessage(
                    content="Add a chart section",
                    ui_context=MaxUIContext(
                        notebooks=[
                            MaxNotebookContext(
                                id=notebook.short_id,
                                insertion_placeholder_block_id="inline-request",
                                insertion_placeholder_marker="Thinking...",
                            )
                        ]
                    ),
                )
            ]
        )
        _, artifact = async_to_sync(self.tool._arun_impl)(
            title="Chart notes",
            content='# Chart notes\n\n<SQLV2 code="SELECT 1" />',
            artifact_id=notebook.short_id,
            save_to_notebook=save_to_notebook,
        )
        assert artifact is not None
        assert Notebook.objects.filter(team=self.team).count() == 1
        assert notebook.short_id in artifact.messages[1].content
        assert "No separate notebook was saved" in artifact.messages[1].content

    def test_save_to_notebook_uses_artifact_short_id(self):
        result, artifact = async_to_sync(self.tool._arun_impl)(
            title="Linked Notebook",
            content="# Hello",
            save_to_notebook=True,
        )

        assert artifact is not None
        from products.posthog_ai.backend.models.assistant import AgentArtifact

        agent_artifact = AgentArtifact.objects.filter(team=self.team).last()
        notebook = Notebook.objects.filter(team=self.team).first()
        assert notebook is not None
        assert agent_artifact is not None
        assert notebook.short_id == agent_artifact.short_id

    @parameterized.expand([("allowed", False), ("denied", True)])
    @patch("posthoganalytics.feature_enabled", return_value=True)
    def test_update_already_saved_notebook_auto_updates_db(
        self, _name: str, denied: bool, _feature_enabled: Mock
    ) -> None:
        # First create and save
        _, create_artifact = async_to_sync(self.tool._arun_impl)(
            title="Original",
            content="# Original",
            save_to_notebook=True,
        )
        short_id = create_artifact.messages[1].content.split("short_id: ")[1].split(".")[0]
        notebook = Notebook.objects.get(team=self.team, short_id=short_id)
        next_content = "# Updated Content"
        if denied:
            self.organization.available_product_features = [
                {"key": AvailableFeature.ACCESS_CONTROL, "name": AvailableFeature.ACCESS_CONTROL}
            ]
            self.organization.save()
            membership = OrganizationMembership.objects.get(user=self.user, organization=self.organization)
            membership.level = OrganizationMembership.Level.MEMBER
            membership.save()
            DataWarehouseSavedQuery.objects.create(
                team=self.team,
                name="weekly_totals",
                query={"kind": "HogQLQuery", "query": "SELECT 3 AS total"},
                columns={"total": "Integer"},
            )
            AccessControl.objects.create(team=self.team, resource="warehouse_objects", access_level="none")
            SharingConfiguration.objects.create(team=self.team, notebook=notebook, enabled=True)
            next_content = (
                '<Query nodeId="totals" query={{"kind":"HogQLQuery","query":"SELECT total FROM weekly_totals"}} />'
            )

        # Now update without save_to_notebook -- should auto-update because already saved
        result, update_artifact = async_to_sync(self.tool._arun_impl)(
            title="Updated",
            content=next_content,
            artifact_id=short_id,
        )

        if denied:
            assert update_artifact is None
            assert "weekly_totals" in result
            assert "notebook was not changed" in result
            original_content = notebook.content
            original_version = notebook.version
            notebook.refresh_from_db()
            assert notebook.title == "Original"
            assert notebook.content == original_content
            assert notebook.version == original_version
            return

        assert update_artifact is not None
        tool_call_message = update_artifact.messages[1]
        assert "updated" in tool_call_message.content.lower()

        notebook = Notebook.objects.get(team=self.team, short_id=short_id)
        assert notebook.title == "Updated"