Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

49.9 KB
Raw Blame
import { afterEach, beforeAll, describe, expect, it, vi, type MockInstance } from 'vitest'
import { z } from 'zod'

vi.mock('@/resources/internals', () => ({
    fetchContextMillResources: vi.fn().mockRejectedValue(new Error('mocked')),
    filterValidEntries: vi.fn().mockReturnValue([]),
    loadManifestFromArchive: vi.fn().mockReturnValue({ resources: [] }),
}))

vi.mock('@/resources', () => ({
    getPromptsFromManifest: vi.fn().mockResolvedValue([]),
}))

import { MCP_EXEC_SKILLS_FEATURE_FLAG } from '@/hono/constants'
import { InstructionsBuilder } from '@/hono/instructions'
import type { ResolvedState } from '@/hono/request-state-resolver'
import { ToolCatalog } from '@/hono/tool-catalog'
import { ToolExecutor } from '@/hono/tool-executor'
import { MCPClientProfile } from '@/lib/client-detection'
import { PostHogApiError, PostHogPermissionError } from '@/lib/errors'
import { buildToolDomainsCompact } from '@/lib/instructions'
import { CHATGPT_APP_OAUTH_CLIENT_ID } from '@/lib/oauth-constants'
import { RENDER_UI_RESOURCE_URI, URI_MAP } from '@/resources/ui-apps.generated'
import { makeSkillFile, SkillCatalog } from '@/skills/skill-catalog'
import { GENERATED_TOOL_MAP } from '@/tools/generated'
import { getToolDefinition } from '@/tools/toolDefinitions'
import { POSTHOG_FORMATTED_RESULTS_OVERRIDE_KEY } from '@/tools/types'
import { APP_DATA_META_KEY } from '@/ui-apps/types'

import { makeToolExecutorState, mockApi } from '../shared/test-utils'

// A tool with a renderable (dispatchable) UI app — used to exercise the render-ui path.
const uiAppTool = {
    name: 'survey-get',
    annotations: { readOnlyHint: true },
    _meta: { ui: { resourceUri: URI_MAP['survey'] } },
}

describe('ToolExecutor', () => {
    let catalog: ToolCatalog
    let executor: ToolExecutor

    beforeAll(async () => {
        catalog = new ToolCatalog()
        await catalog.warmup()
        executor = new ToolExecutor(catalog, new InstructionsBuilder(''))
    })

    describe('handleToolCall', () => {
        it('refuses cached exec calls through Store connections', async () => {
            const result = await executor.handleToolCall(
                { name: 'exec', arguments: { command: 'tools' } },
                makeToolExecutorState([], { useSingleExec: false, viaMcpStore: true })
            )
            expect(result).toMatchObject({ isError: true, content: [{ type: 'text', text: 'Tool exec not found' }] })
        })

        it('returns error when tool name is missing', async () => {
            const result = (await executor.handleToolCall({}, makeToolExecutorState([]))) as any
            expect(result.isError).toBe(true)
            expect(result.content[0].text).toContain('Missing tool name')
        })

        it('returns error when tool does not exist', async () => {
            const result = (await executor.handleToolCall(
                { name: 'nonexistent-tool', arguments: {} },
                makeToolExecutorState([])
            )) as any
            expect(result.isError).toBe(true)
            expect(result.content[0].text).toContain('nonexistent-tool')
            expect(result.content[0].text).toContain('not found')
        })

        it('returns replacement guidance for a direct call to a removed tool', async () => {
            const result = (await executor.handleToolCall(
                { name: 'loops-list', arguments: {} },
                makeToolExecutorState([])
            )) as any
            expect(result.isError).toBe(true)
            expect(result.content[0].text).toContain('loops-list')
            expect(result.content[0].text).toContain('workflows-list')
            expect(result.content[0].text).toContain('building-loops')
        })

        it('rejects tools not in the per-request filtered set', async () => {
            const entries = catalog.getPreBuiltEntries()
            const tool = entries[0]!

            const result = (await executor.handleToolCall(
                { name: tool.name, arguments: {} },
                makeToolExecutorState([])
            )) as any
            expect(result.isError).toBe(true)
            expect(result.content[0].text).toContain('not found')
        })

        it('returns validation error for invalid arguments', async () => {
            const knownTool = catalog.getPreBuiltEntries()[0]
            if (!knownTool) {
                throw new Error('need at least one tool to test validation')
            }

            const result = (await executor.handleToolCall(
                { name: knownTool.name, arguments: { __invalid_field_xyz: 'bad' } },
                makeToolExecutorState([{ name: knownTool.name }])
            )) as any

            expect(result).not.toBeNull()
        })

        it('successfully calls a real tool from the catalog', async () => {
            const entries = catalog.getPreBuiltEntries()
            const userGet = entries.find((e) => e.name === 'user-get')
            if (!userGet) {
                throw new Error('user-get tool not found in catalog')
            }

            const result = (await executor.handleToolCall(
                { name: 'user-get', arguments: {} },
                makeToolExecutorState([{ name: 'user-get' }])
            )) as any

            expect(result).not.toBeNull()
            expect(result.content).not.toBeNull()
        })

        it('accepts cached exec calls even when the current session is in tools mode', async () => {
            const filteredTools = catalog
                .getFilteredTools({ scopes: ['*'] })
                .filter((tool) => tool.name === 'execute-sql' || tool.name === 'organization-get')

            const result = (await executor.handleToolCall(
                { name: 'exec', arguments: { command: 'tools' } },
                makeToolExecutorState(filteredTools, { useSingleExec: false })
            )) as any

            expect(result.isError).toBeFalsy()
            const text = result.content?.[0]?.text ?? ''
            expect(text).toContain('execute-sql')
            expect(text).toContain('organization-get')
            expect(text).not.toContain('feature-flag-get-all')
        })

        function skillMissContext(skillName: string, body: string): ResolvedState['context'] {
            return {
                api: mockApi({
                    request: vi.fn().mockRejectedValue(
                        new PostHogApiError({
                            status: 404,
                            statusText: 'Not Found',
                            body,
                            url: `https://internal.example.com/api/projects/1/llm_skills/name/${skillName}/`,
                            method: 'GET',
                        })
                    ),
                }),
                cache: {},
                env: {},
                stateManager: { getProjectId: vi.fn().mockResolvedValue('1') },
                sessionManager: {},
                getDistinctId: vi.fn(),
                trackEvent: vi.fn(),
            } as any
        }

        // Cursor and ChatGPT get the per-tool roster, so they call the skill read
        // tools here rather than through exec. Left on the generic error shape, the
        // same miss reads as a service outage for them and as a plain answer for
        // everyone else — the divergence this rewrite exists to remove.
        it.each([
            [
                'skill-get',
                { skill_name: 'missing-skill' },
                '{"detail":"Skill with name \'missing-skill\' not found."}',
                'No skill named "missing-skill" in this project\'s skills store.',
            ],
            [
                'skill-file-get',
                { skill_name: 'real-skill', file_path: 'refs/guide.md' },
                '{"detail":"File \'refs/guide.md\' not found in skill \'real-skill\'."}',
                'No file "refs/guide.md" in the skill "real-skill".',
            ],
        ])('answers a %s miss with the plain message in tools mode', async (name, args, body, expected) => {
            const state = makeToolExecutorState([{ name }], { context: skillMissContext(args.skill_name, body) })

            const result = (await executor.handleToolCall({ name, arguments: args }, state)) as any

            expect(result.isError).toBeFalsy()
            expect(result.content[0].text).toContain(expected)
            expect(result.content[0].text).not.toContain('Request failed')
        })

        // The `learn` command is feature-flagged, so most connections cannot load a
        // built-in skill at all. Sending them back to a store that never held one
        // is what makes an agent drop the task, so the message says where the skill
        // lives and whether this connection can reach it.
        // Both modes are covered: single-exec dispatch is where nearly every skill
        // read arrives, and the per-tool roster is the path Cursor and ChatGPT take.
        it.each([
            ['tools', false, 'this connection cannot load built-in skills.'],
            ['tools', true, 'Run `learn posthog:scanning-experiments-with-replay-vision` to load it.'],
            ['exec', false, 'this connection cannot load built-in skills.'],
            ['exec', true, 'Run `learn posthog:scanning-experiments-with-replay-vision` to load it.'],
        ])(
            'names the built-in catalog on a store miss in %s mode, with learn enabled=%s',
            async (mode, skillsEnabled, expected) => {
                const skillName = 'scanning-experiments-with-replay-vision'
                const skills = new SkillCatalog([
                    {
                        name: skillName,
                        description: 'A built-in skill.',
                        files: [makeSkillFile('SKILL.md', '# Built-in skill')],
                    },
                ])
                const skillExecutor = new ToolExecutor(catalog, new InstructionsBuilder(''), {
                    getCatalog: () => skills,
                } as any)
                const useExec = mode === 'exec'
                // Exec dispatches through the real tool objects, so the roster has to
                // carry the handler rather than only the name.
                const tools = useExec
                    ? catalog.getFilteredTools({ scopes: ['*'] }).filter((tool) => tool.name === 'skill-get')
                    : [{ name: 'skill-get' }]
                const state = makeToolExecutorState(tools as any, {
                    useSingleExec: useExec,
                    toolFeatureFlags: { [MCP_EXEC_SKILLS_FEATURE_FLAG]: skillsEnabled },
                    context: skillMissContext(skillName, `{"detail":"Skill with name '${skillName}' not found."}`),
                })
                const call = useExec
                    ? { name: 'exec', arguments: { command: `call skill-get {"skill_name":"${skillName}"}` } }
                    : { name: 'skill-get', arguments: { skill_name: skillName } }

                const result = (await skillExecutor.handleToolCall(call, state)) as any

                expect(result.isError).toBeFalsy()
                expect(result.content[0].text).toContain(expected)
                expect(result.content[0].text).not.toContain('call skill-list')
            }
        )
    })

    describe('handleToolsList', () => {
        it('returns filtered tools matching the state allTools', async () => {
            const allEntries = catalog.getPreBuiltEntries()
            const subset = allEntries.slice(0, 3)

            const result = await executor.handleToolsList(makeToolExecutorState(subset.map((e) => ({ name: e.name }))))

            expect(result.tools).toHaveLength(3)
            expect(result.tools.map((t) => t.name)).toEqual(subset.map((e) => e.name))
        })

        it('returns empty list when allTools is empty', async () => {
            const result = await executor.handleToolsList(makeToolExecutorState([]))
            expect(result.tools).toEqual([])
        })

        it('returns single exec tool entry when useSingleExec is true', async () => {
            const state = makeToolExecutorState(
                catalog
                    .getPreBuiltEntries()
                    .slice(0, 5)
                    .map((e) => ({ name: e.name })),
                { useSingleExec: true }
            )

            const result = await executor.handleToolsList(state)
            expect(result.tools).toHaveLength(1)
            expect(result.tools[0]!.name).toBe('exec')
        })

        // The flag decides skill exposure per client; guides stay a Claude chat-host
        // feature; plugin and sandbox consumers get neither because they carry their
        // own skills. A regression in any row silently changes what an agent is told.
        it.each([
            {
                label: 'Claude web/desktop with flag off',
                isClaudeChatHost: true,
                skillsEnabled: false,
                consumer: undefined,
                expectGuides: true,
                expectSkills: false,
            },
            {
                label: 'Claude web/desktop with flag on',
                isClaudeChatHost: true,
                skillsEnabled: true,
                consumer: undefined,
                expectGuides: true,
                expectSkills: true,
            },
            {
                label: 'directly connected Claude Code with flag off',
                isClaudeChatHost: false,
                skillsEnabled: false,
                consumer: undefined,
                expectGuides: false,
                expectSkills: false,
            },
            {
                label: 'directly connected Claude Code with flag on',
                isClaudeChatHost: false,
                skillsEnabled: true,
                consumer: undefined,
                expectGuides: false,
                expectSkills: true,
            },
            {
                label: 'plugin consumer with flag on',
                isClaudeChatHost: true,
                skillsEnabled: true,
                consumer: 'plugin',
                expectGuides: false,
                expectSkills: false,
            },
            {
                label: 'posthog-code consumer with flag on',
                isClaudeChatHost: false,
                skillsEnabled: true,
                consumer: 'posthog-code',
                expectGuides: false,
                expectSkills: false,
            },
        ])(
            'advertises and serves the expected learn capabilities for $label',
            async ({ isClaudeChatHost, skillsEnabled, consumer, expectGuides, expectSkills }) => {
                const skills = new SkillCatalog([
                    {
                        name: 'sample-skill',
                        description: 'A sample skill.',
                        files: [makeSkillFile('SKILL.md', '# Sample skill')],
                    },
                ])
                const skillExecutor = new ToolExecutor(catalog, new InstructionsBuilder(''), {
                    getCatalog: () => skills,
                } as any)
                const state = makeToolExecutorState([], {
                    useSingleExec: true,
                    toolFeatureFlags: { [MCP_EXEC_SKILLS_FEATURE_FLAG]: skillsEnabled },
                    clientProfile: {
                        capabilities: { supportsInstructions: true },
                        isCliModeEnabled: vi.fn(() => true),
                        isInlineExecUiHost: vi.fn(() => false),
                        isClaudeChatHost: vi.fn(() => isClaudeChatHost),
                        isAnthropicConnector: vi.fn(() => isClaudeChatHost),
                    } as any,
                    ...(consumer
                        ? {
                              sessionContext: {
                                  mcpClientName: 'claude-ai',
                                  mcpClientVersion: '1.0',
                                  mcpProtocolVersion: '2025-03-26',
                                  mcpConsumer: consumer,
                                  mcpVendorClient: undefined,
                              },
                          }
                        : {}),
                })

                const listed = await skillExecutor.handleToolsList(state)
                const commandDescription = (listed.tools[0]!.inputSchema.properties as any).command
                    .description as string
                const advertisesSkills =
                    commandDescription.includes('learn posthog:<skill>') ||
                    commandDescription.includes('(posthog|project):<skill>')
                expect(commandDescription.includes('- analytics:')).toBe(expectGuides)
                expect(advertisesSkills).toBe(expectSkills)
                expect(commandDescription.includes('**SKILLS FIRST: HARD REQUIREMENT**')).toBe(expectSkills)
                expect(commandDescription.includes('learn <topic...>')).toBe(expectGuides)
                expect(listed.tools[0]!.description!.includes('SKILLS FIRST')).toBe(expectSkills)

                const result = (await skillExecutor.handleToolCall(
                    { name: 'exec', arguments: { command: 'learn' } },
                    state
                )) as { content: { text: string }[]; isError?: boolean }
                if (!expectGuides && !expectSkills) {
                    expect(result.isError).toBe(true)
                    expect(result.content[0]!.text).toContain('learn command is not available')
                    return
                }

                const catalogResult = JSON.parse(result.content[0]!.text)
                expect(catalogResult.guides.length > 0).toBe(expectGuides)
                expect('skills' in catalogResult).toBe(expectSkills)

                if (!expectSkills) {
                    const skillResult = (await skillExecutor.handleToolCall(
                        { name: 'exec', arguments: { command: 'learn skills' } },
                        state
                    )) as { content: { text: string }[]; isError?: boolean }
                    expect(skillResult.isError).toBeFalsy()
                    expect(JSON.parse(skillResult.content[0]!.text)).toEqual({
                        available: false,
                        reason: 'Skill discovery is not enabled for this connection.',
                    })
                }
            }
        )

        it.each([
            { vendorClient: 'ClaudeCode', skillsEnabled: false },
            { vendorClient: 'ClaudeCode', skillsEnabled: true },
            { vendorClient: 'Cowork', skillsEnabled: false },
            { vendorClient: 'Cowork', skillsEnabled: true },
        ])(
            'serves the advertised guides when the call names $vendorClient (skills flag: $skillsEnabled)',
            async ({ vendorClient, skillsEnabled }) => {
                const tools = catalog
                    .getPreBuiltEntries()
                    .slice(0, 5)
                    .map(({ name }) => ({ name }))
                const connectorState = (profile: MCPClientProfile): ResolvedState =>
                    makeToolExecutorState(tools, {
                        useSingleExec: true,
                        toolFeatureFlags: { [MCP_EXEC_SKILLS_FEATURE_FLAG]: skillsEnabled },
                        clientProfile: profile,
                    })
                const listState = connectorState(
                    new MCPClientProfile({ clientName: 'Anthropic/ClaudeAI', userAgent: 'Claude-User' })
                )
                const callState = connectorState(
                    new MCPClientProfile({ clientName: 'Anthropic/ClaudeAI', vendorClient, userAgent: 'Claude-User' })
                )

                const listed = await executor.handleToolsList(listState)
                const commandDescription = (listed.tools[0]!.inputSchema.properties as any).command
                    .description as string
                expect(commandDescription).toContain('- analytics:')

                const result = (await executor.handleToolCall(
                    { name: 'exec', arguments: { command: 'learn analytics' } },
                    callState
                )) as { content: { text: string }[]; isError?: boolean }
                expect(result.isError).toBeFalsy()
                expect(result.content[0]!.text).toContain('### Retrieving data')
            }
        )

        it.each([{ skillsEnabled: false }, { skillsEnabled: true }])(
            'keeps guides off a directly connected Claude Code CLI (skills flag: $skillsEnabled)',
            async ({ skillsEnabled }) => {
                const state = makeToolExecutorState(
                    catalog
                        .getPreBuiltEntries()
                        .slice(0, 5)
                        .map(({ name }) => ({ name })),
                    {
                        useSingleExec: true,
                        toolFeatureFlags: { [MCP_EXEC_SKILLS_FEATURE_FLAG]: skillsEnabled },
                        clientProfile: new MCPClientProfile({ clientName: 'claude-code', vendorClient: 'ClaudeCode' }),
                    }
                )

                const result = (await executor.handleToolCall(
                    { name: 'exec', arguments: { command: 'learn analytics' } },
                    state
                )) as { content: { text: string }[]; isError?: boolean }
                expect(result.isError).toBe(true)
                expect(result.content[0]!.text).toContain(
                    skillsEnabled ? 'Unknown learning topic' : 'learn command is not available'
                )
            }
        )

        it('discovers and searches current-project skills when the connection has read scope', async () => {
            const apiRequest = vi.fn().mockImplementation(async ({ path }: { path: string }) => {
                if (path.endsWith('/search/')) {
                    return {
                        count: 1,
                        results: [
                            {
                                name: 'team-retention',
                                description: 'Project-specific retention guidance.',
                                score: 900,
                                matches: [
                                    {
                                        matched_field: 'body',
                                        path: 'SKILL.md',
                                        line: 3,
                                        excerpt: 'Use weekly retention cohorts.',
                                    },
                                ],
                            },
                        ],
                    }
                }
                return { count: 1, results: [{ name: 'team-retention' }] }
            })
            const state = makeToolExecutorState([], {
                useSingleExec: true,
                toolFeatureFlags: { [MCP_EXEC_SKILLS_FEATURE_FLAG]: true },
                apiKeyScopes: ['llm_skill:read'],
                context: {
                    api: mockApi({ request: apiRequest }),
                    stateManager: { getProjectId: vi.fn().mockResolvedValue(12) },
                } as any,
            })

            const result = (await executor.handleToolCall(
                { name: 'exec', arguments: { command: 'learn -s retention' } },
                state
            )) as { content: { text: string }[] }

            expect(result.content[0]!.text).toContain('## project:team-retention')
            expect(apiRequest).toHaveBeenCalledWith({
                method: 'GET',
                path: '/api/projects/12/llm_skills/search/',
                query: { query: 'retention' },
            })

            const listResult = (await executor.handleToolCall(
                { name: 'exec', arguments: { command: 'learn skills' } },
                state
            )) as { content: { text: string }[] }
            expect(listResult.content[0]!.text).toContain('project:team-retention')
            expect(apiRequest).toHaveBeenCalledWith({
                method: 'GET',
                path: '/api/projects/12/llm_skills/',
                query: { category: '', limit: 100, offset: 0, order_by: 'name' },
            })
        })

        it.each([
            {
                connection: 'the PostHog app for ChatGPT and Codex',
                oauthClientId: CHATGPT_APP_OAUTH_CLIENT_ID,
                reconnectHint: true,
            },
            {
                connection: "the Codex CLI's own OAuth client",
                oauthClientId: 'https://chatgpt.com/oauth/codex/51XaKixG06mz/client.json',
                reconnectHint: false,
            },
            { connection: 'a personal API key', oauthClientId: undefined, reconnectHint: false },
        ])(
            'tells the agent project skills need the read scope, with a reconnect step only for $connection',
            async ({ oauthClientId, reconnectHint }) => {
                const state = makeToolExecutorState([], {
                    useSingleExec: true,
                    toolFeatureFlags: { [MCP_EXEC_SKILLS_FEATURE_FLAG]: true },
                    apiKeyScopes: ['insight:read'],
                    oauthClientId,
                })

                const result = (await executor.handleToolCall(
                    { name: 'exec', arguments: { command: 'learn skills' } },
                    state
                )) as { content: { text: string }[] }

                const { project } = JSON.parse(result.content[0]!.text)
                expect(project.available).toBe(false)
                expect(project.reason).toContain('llm_skill:read')
                if (reconnectHint) {
                    expect(project.reason).toContain('disconnect the PostHog app in ChatGPT or Codex')
                } else {
                    expect(project.reason).not.toMatch(/ChatGPT|Codex/)
                    expect(project.reason).toContain('Reconnect with that scope')
                }
            }
        )

        // Hosts cache one tool roster and serve it to other accounts, so nothing
        // account-specific may reach the advertised exec entry. The domain index stays
        // out of the command description except for Claude web/desktop, which ignores
        // `instructions` and has nowhere else to receive it.
        it.each([
            {
                label: 'Claude web/desktop (ignores instructions)',
                supportsInstructions: true,
                isClaudeChatHost: true,
            },
            {
                label: 'Codex (supportsInstructions: false)',
                supportsInstructions: false,
                isClaudeChatHost: false,
            },
            {
                label: 'Claude Code / Cowork (consume instructions)',
                supportsInstructions: true,
                isClaudeChatHost: false,
            },
        ])(
            'advertises the same exec entry to every account for $label',
            async ({ supportsInstructions, isClaudeChatHost }) => {
                const tools = [
                    ...catalog
                        .getPreBuiltEntries()
                        .slice(0, 5)
                        .map((e) => ({ name: e.name })),
                    { name: 'project-get' },
                ]
                const stateFor = (distinctId: string): ReturnType<typeof makeToolExecutorState> =>
                    makeToolExecutorState(tools, {
                        useSingleExec: true,
                        distinctId,
                        clientProfile: {
                            capabilities: { supportsInstructions },
                            isCliModeEnabled: vi.fn(() => true),
                            isInlineExecUiHost: vi.fn(() => false),
                            isClaudeChatHost: vi.fn(() => isClaudeChatHost),
                            isAnthropicConnector: vi.fn(() => isClaudeChatHost),
                        } as any,
                    })

                const result = await executor.handleToolsList(stateFor('account-a'))
                const other = await executor.handleToolsList(stateFor('account-b'))
                const commandDesc = (result.tools[0]!.inputSchema.properties as any).command.description as string
                const compactDomains = buildToolDomainsCompact(
                    tools.map(({ name }) => ({ name, category: getToolDefinition(name).category }))
                )

                expect(JSON.stringify(other.tools)).toBe(JSON.stringify(result.tools))
                expect(commandDesc).toContain('PostHog tools have lowercase kebab-case naming')
                expect(commandDesc.includes('**LEARN FIRST: HARD REQUIREMENT**')).toBe(isClaudeChatHost)
                expect(commandDesc.includes('- analytics:')).toBe(isClaudeChatHost)
                expect(commandDesc.includes('### Retrieving data')).toBe(!isClaudeChatHost)
                expect(commandDesc.includes(compactDomains)).toBe(isClaudeChatHost)
                expect(commandDesc).toContain('Call `project-get` without an ID to read the active project')
            }
        )

        it('serves multiple optional guidance topics through exec learn for Claude web/desktop', async () => {
            const state = makeToolExecutorState(
                catalog
                    .getPreBuiltEntries()
                    .slice(0, 5)
                    .map(({ name }) => ({ name })),
                {
                    useSingleExec: true,
                    renderUiEnabled: true,
                    clientProfile: {
                        capabilities: { supportsInstructions: true },
                        isCliModeEnabled: vi.fn(() => true),
                        isInlineExecUiHost: vi.fn(() => false),
                        isClaudeChatHost: vi.fn(() => true),
                        isAnthropicConnector: vi.fn(() => true),
                    } as any,
                }
            )

            const result = (await executor.handleToolCall(
                { name: 'exec', arguments: { command: 'learn analytics visualizations' } },
                state
            )) as { content: { text: string }[] }

            expect(result.content[0]!.text).toContain('## Analytics')
            expect(result.content[0]!.text).toContain('### Retrieving data')
            expect(result.content[0]!.text).toContain('### Examples')
            expect(result.content[0]!.text).toContain('## Visualizations')
            expect(result.content[0]!.text).toContain('### Rendering visualizations')
        })

        it('advertises the same render-ui schema across accounts with different allowed tools', async () => {
            const state = makeToolExecutorState([uiAppTool], { useSingleExec: true, renderUiEnabled: true })

            const result = await executor.handleToolsList(state)
            expect(result.tools.map((t) => t.name)).toEqual(['exec', 'render-ui'])

            // Pin the complete advertised schema because clients generate
            // render-ui calls from this list response.
            const renderUiEntry = result.tools[1]!
            const properties = renderUiEntry.inputSchema.properties as Record<string, Record<string, unknown>>
            expect(properties.tool_name!.enum).toEqual(expect.arrayContaining(['survey-get', 'query-trends']))
            expect(properties.tool_name!.enum).not.toContain('survey-launch')
            expect(properties.tool_name!.description).toBeTruthy()
            expect(properties.tool_input!.description).toBeTruthy()
            expect(properties.context!.description).toBeTruthy()
            expect(properties.llm_model!.description).toBeTruthy()
            expect(renderUiEntry.inputSchema.required).toEqual(['tool_name', 'context', 'llm_model'])
            expect(renderUiEntry.annotations).toEqual({
                destructiveHint: false,
                idempotentHint: true,
                openWorldHint: false,
                readOnlyHint: true,
            })

            const restrictedResult = await executor.handleToolsList(
                makeToolExecutorState([], { useSingleExec: true, renderUiEnabled: true })
            )
            expect(restrictedResult.tools.find((tool) => tool.name === 'render-ui')).toEqual(renderUiEntry)
        })

        it('omits render-ui when render-ui is disabled, even with a UI-app tool available', async () => {
            const state = makeToolExecutorState([uiAppTool], { useSingleExec: true, renderUiEnabled: false })

            const result = await executor.handleToolsList(state)
            expect(result.tools.map((t) => t.name)).toEqual(['exec'])
        })
    })

    describe('render-ui', () => {
        it('dispatches to the render-ui payload when render-ui is enabled', async () => {
            const state = makeToolExecutorState([uiAppTool], { useSingleExec: true, renderUiEnabled: true })

            const result = (await executor.handleToolCall(
                { name: 'render-ui', arguments: { tool_name: 'survey-get', tool_input: { surveyId: 'abc' } } },
                state
            )) as any

            expect(result._meta.ui.resourceUri).toBe(RENDER_UI_RESOURCE_URI)
            expect(result.structuredContent.tool_name).toBe('survey-get')
            expect(result.structuredContent.app_key).toBe('survey')
        })

        it('rejects a render-ui call when render-ui is disabled', async () => {
            const state = makeToolExecutorState([uiAppTool], { useSingleExec: true, renderUiEnabled: false })

            const result = (await executor.handleToolCall(
                { name: 'render-ui', arguments: { tool_name: 'survey-get', tool_input: { surveyId: 'abc' } } },
                state
            )) as any

            expect(result.isError).toBe(true)
            expect(result.content[0].text).toContain('not found')
        })

        // Some hosts (Claude Code 2.1.308+) never forward the app's own `tools/call`, so the
        // first view must draw from data the render-ui result already carries.
        describe('preloaded app data', () => {
            let getToolByNameSpy: MockInstance | undefined

            afterEach(() => {
                getToolByNameSpy?.mockRestore()
                getToolByNameSpy = undefined
            })

            function stubInnerTool(handler: () => Promise<unknown>): void {
                getToolByNameSpy = vi.spyOn(catalog, 'getToolByName').mockReturnValue({
                    build() {
                        return this.base
                    },
                    base: { schema: z.object({ surveyId: z.string() }), handler, _meta: uiAppTool._meta },
                } as any)
            }

            async function renderSurvey(): Promise<any> {
                const state = makeToolExecutorState([uiAppTool], { useSingleExec: true, renderUiEnabled: true })
                return executor.handleToolCall(
                    { name: 'render-ui', arguments: { tool_name: 'survey-get', tool_input: { surveyId: 'abc' } } },
                    state
                )
            }

            it('carries the data the app would fetch on app-only _meta', async () => {
                stubInnerTool(async () => ({ id: 'abc', name: 'NPS' }))

                const result = await renderSurvey()

                expect(result._meta[APP_DATA_META_KEY]).toMatchObject({ id: 'abc', name: 'NPS' })
                expect(result.structuredContent).not.toHaveProperty('name')
            })

            it('preloads query charts through exec as JSON, like the app does', async () => {
                const data = { results: [{ label: '$pageview', data: [3] }] }
                // Exec runs the tools on the state's roster, so the stub carries a schema and handler.
                const queryTool = {
                    name: 'query-trends',
                    annotations: { readOnlyHint: true },
                    _meta: { ui: { resourceUri: URI_MAP['query-results'] } },
                    schema: z.object({}),
                    handler: async () => data,
                }
                const state = makeToolExecutorState([queryTool], { useSingleExec: true, renderUiEnabled: true })

                const result = (await executor.handleToolCall(
                    { name: 'render-ui', arguments: { tool_name: 'query-trends', tool_input: {} } },
                    state
                )) as any

                expect(result._meta[APP_DATA_META_KEY]).toEqual(data)
            })

            it('preloads through a normal tool call, so input validation still applies', async () => {
                const handler = vi.fn(async () => ({ id: 'abc' }))
                stubInnerTool(handler)
                const state = makeToolExecutorState([uiAppTool], { useSingleExec: true, renderUiEnabled: true })

                const result = (await executor.handleToolCall(
                    { name: 'render-ui', arguments: { tool_name: 'survey-get', tool_input: { surveyId: 123 } } },
                    state
                )) as any

                expect(handler).not.toHaveBeenCalled()
                expect(result.isError).toBeFalsy()
                expect(result._meta).not.toHaveProperty(APP_DATA_META_KEY)
            })

            it('still renders, leaving the app to fetch, when the preload fails', async () => {
                stubInnerTool(async () => {
                    throw new Error('boom')
                })

                const result = await renderSurvey()

                expect(result.isError).toBeFalsy()
                expect(result.structuredContent.tool_name).toBe('survey-get')
                expect(result._meta).not.toHaveProperty(APP_DATA_META_KEY)
            })

            it('stops waiting for a slow preload before the app gives up on the result', async () => {
                vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] })
                try {
                    stubInnerTool(() => new Promise(() => {}))

                    const pending = renderSurvey()
                    await vi.advanceTimersByTimeAsync(15_000)
                    const result = await pending

                    expect(result._meta).not.toHaveProperty(APP_DATA_META_KEY)
                } finally {
                    vi.useRealTimers()
                }
            })
        })

        it('rejects a cached render-ui choice that is not allowed for the current account', async () => {
            const state = makeToolExecutorState([uiAppTool], { useSingleExec: true, renderUiEnabled: true })

            const result = (await executor.handleToolCall(
                { name: 'render-ui', arguments: { tool_name: 'query-trends', tool_input: {} } },
                state
            )) as { isError?: boolean; _meta?: { ui?: unknown } }

            expect(result.isError).toBe(true)
            expect(result._meta?.ui).toBeUndefined()
        })
    })

    // A render-ui app loads its own data with a direct `tools/call`, reading
    // `structuredContent` and ignoring the text channel. The formatted-table
    // suppression that serves text-reading CLI clients must not reach that fetch,
    // or the app has nothing to draw and shows its error state instead of the chart.
    describe('structuredContent on a UI-app data fetch', () => {
        const formattedTable = 'date|count\n2026-08-31|28'
        let getToolByNameSpy: MockInstance | undefined

        afterEach(() => {
            getToolByNameSpy?.mockRestore()
            getToolByNameSpy = undefined
        })

        it.each([
            {
                label: 'keeps it for a render-ui host, where this call is the app',
                useSingleExec: true,
                renderUiEnabled: true,
                expectStructuredContent: true,
                posthogAi: false,
            },
            {
                label: 'drops it for a CLI client without render-ui, which reads the table',
                useSingleExec: true,
                renderUiEnabled: false,
                expectStructuredContent: false,
                posthogAi: false,
            },
            {
                label: 'drops it in tools mode, where a direct call may be the model',
                useSingleExec: false,
                renderUiEnabled: true,
                expectStructuredContent: false,
                posthogAi: false,
            },
            {
                label: 'carries app data for native widgets without a UI resource',
                useSingleExec: false,
                renderUiEnabled: false,
                expectStructuredContent: false,
                posthogAi: true,
            },
        ])('$label', async ({ useSingleExec, renderUiEnabled, expectStructuredContent, posthogAi }) => {
            getToolByNameSpy = vi.spyOn(catalog, 'getToolByName').mockReturnValue({
                build() {
                    return this.base
                },
                base: {
                    schema: z.object({}),
                    handler: async () => ({
                        results: [{ count: 28, label: '$pageview' }],
                        [POSTHOG_FORMATTED_RESULTS_OVERRIDE_KEY]: formattedTable,
                    }),
                    _meta: posthogAi ? undefined : uiAppTool._meta,
                },
            } as any)

            const state = makeToolExecutorState([uiAppTool], { useSingleExec, renderUiEnabled })
            vi.mocked(state.clientProfile.isCliModeEnabled).mockReturnValue(true)
            if (posthogAi) {
                state.clientProfile = { ...state.clientProfile, consumer: 'posthog_ai' } as typeof state.clientProfile
            }

            const result = (await executor.handleToolCall({ name: 'survey-get', arguments: {} }, state)) as any

            expect(result.content[0].text).toContain(formattedTable)
            expect('structuredContent' in result).toBe(expectStructuredContent)
            if (posthogAi) {
                expect(result._meta['com.posthog.mcp/app_data']).toEqual({
                    results: [{ count: 28, label: '$pageview' }],
                })
            }
        })
    })

    describe('a query wrapper called with its payload nested under `query`', () => {
        let getToolByNameSpy: MockInstance | undefined

        afterEach(() => {
            getToolByNameSpy?.mockRestore()
            getToolByNameSpy = undefined
        })

        it('runs query-trends with the payload lifted to the top level', async () => {
            const received: unknown[] = []
            const { schema } = GENERATED_TOOL_MAP['query-trends']!()
            getToolByNameSpy = vi.spyOn(catalog, 'getToolByName').mockReturnValue({
                build() {
                    return this.base
                },
                base: {
                    schema,
                    handler: async (_context: unknown, params: unknown) => (received.push(params), { results: [] }),
                },
            } as any)
            const payload = { series: [{ kind: 'EventsNode', event: '$pageview' }], dateRange: { date_from: '-7d' } }

            const result = (await executor.handleToolCall(
                { name: 'query-trends', arguments: { query: payload } },
                makeToolExecutorState([{ name: 'query-trends' }], { useSingleExec: false })
            )) as any

            expect(result.isError).toBeFalsy()
            expect(received).toEqual([schema.parse(payload)])
        })
    })

    // A tools-mode client calls the metric-run tool directly, bypassing the exec
    // dispatcher that marks the result. Both paths have to agree, or whether an agent
    // is warned off an unapproved metric depends on the client it runs in.
    describe('governed metric run canonicality in tools mode', () => {
        const metricRunTool = { name: 'data-catalog-metric-run' }
        let getToolByNameSpy: MockInstance | undefined

        afterEach(() => {
            getToolByNameSpy?.mockRestore()
            getToolByNameSpy = undefined
        })

        function stubMetricRun(envelope: Record<string, unknown>): void {
            getToolByNameSpy = vi.spyOn(catalog, 'getToolByName').mockReturnValue({
                build() {
                    return this.base
                },
                base: {
                    schema: z.object({}),
                    handler: async () => envelope,
                },
            } as any)
        }

        it.each([
            { label: 'proposed', envelope: { status: 'proposed', is_drifted: false }, marked: true },
            { label: 'drifted approved', envelope: { status: 'approved', is_drifted: true }, marked: true },
            { label: 'approved', envelope: { status: 'approved', is_drifted: false }, marked: false },
        ])('$label metric result is marked: $marked', async ({ envelope, marked }) => {
            stubMetricRun({ ...envelope, results: [[42]] })

            const result = (await executor.handleToolCall(
                { name: metricRunTool.name, arguments: {} },
                makeToolExecutorState([metricRunTool], { useSingleExec: false })
            )) as any

            expect(result.content[0].text.includes('NONCANONICAL')).toBe(marked)
        })
    })

    describe('a missing scope reported by the API', () => {
        let getToolByNameSpy: MockInstance | undefined

        afterEach(() => {
            getToolByNameSpy?.mockRestore()
            getToolByNameSpy = undefined
        })

        function stubToolDeniedWith(error: PostHogPermissionError): void {
            getToolByNameSpy = vi.spyOn(catalog, 'getToolByName').mockReturnValue({
                build() {
                    return this.base
                },
                base: {
                    schema: z.object({}),
                    handler: async () => {
                        throw error
                    },
                },
            } as any)
        }

        // The client replays the request after re-consent, so a scope the API reports missing
        // after the tool started must not become a step-up 403.
        it('returns the plain tool error to a step-up token when the API reports a missing scope mid-run', async () => {
            stubToolDeniedWith(
                new PostHogPermissionError({
                    detail: "API key missing required scope 'person:read'",
                    missingScope: 'person:read',
                    challengeScopes: ['person:read', 'cohort:read'],
                    url: 'https://us.posthog.com/api/projects/1/persons/cohorts/',
                    method: 'GET',
                })
            )

            const result = (await executor.handleToolCall(
                { name: 'mock-tool', arguments: {} },
                makeToolExecutorState([{ name: 'mock-tool' }], { supportsStepUp: true })
            )) as any

            expect(result.isError).toBe(true)
            expect(result.content[0].text).toContain("Missing PostHog API scope: 'person:read'")
            expect(result._meta).toBeUndefined()
        })
    })

    describe('a tool that needs a scope the token lacks', () => {
        const scopeGatedTools = [
            {
                name: 'endpoint-create',
                title: 'Create endpoint',
                description: 'Create a new endpoint',
                missingScopes: ['endpoint:write'],
                requiredScopes: ['endpoint:write', 'query:read'],
            },
        ]
        const gated = {
            scopeGatedTools,
            findScopeGatedTool: (name: string) => scopeGatedTools.find((tool) => tool.name === name),
        }
        const calls = [
            { mode: 'tools', useSingleExec: false, params: { name: 'endpoint-create', arguments: {} } },
            {
                mode: 'exec',
                useSingleExec: true,
                params: { name: 'exec', arguments: { command: 'call endpoint-create {}' } },
            },
        ]

        it.each(calls)('throws the step-up challenge before the tool runs in $mode mode', async (call) => {
            const result = executor.handleToolCall(
                call.params,
                makeToolExecutorState([], { useSingleExec: call.useSingleExec, ...gated, supportsStepUp: true })
            )

            await expect(result).rejects.toMatchObject({
                challenge: 'Bearer error="insufficient_scope", scope="endpoint:write query:read"',
            })
        })

        it.each([
            ...calls.map((call) => ({ ...call, supportsStepUp: false })),
            {
                mode: 'exec info',
                useSingleExec: true,
                params: { name: 'exec', arguments: { command: 'info endpoint-create' } },
                supportsStepUp: true,
            },
        ])('returns the plain tool error in $mode mode', async (call) => {
            const result = (await executor.handleToolCall(
                call.params,
                makeToolExecutorState([], {
                    useSingleExec: call.useSingleExec,
                    ...gated,
                    supportsStepUp: call.supportsStepUp,
                })
            )) as any

            expect(result.isError).toBe(true)
            expect(result.content[0].text).toContain('missing the required scope(s): endpoint:write')
            expect(result._meta).toBeUndefined()
        })
    })

    describe('ignored input keys in tools mode', () => {
        let getToolByNameSpy: MockInstance | undefined

        afterEach(() => {
            getToolByNameSpy?.mockRestore()
            getToolByNameSpy = undefined
        })

        it.each([
            { label: 'an unknown top-level key', args: { name: 'a', title: 'b' }, reported: true },
            { label: 'an unknown nested key', args: { name: 'a', query: { kind: 'x', extra: 1 } }, reported: true },
            { label: 'only declared keys', args: { name: 'a', query: { kind: 'x' } }, reported: false },
        ])('$label: reported is $reported', async ({ args, reported }) => {
            getToolByNameSpy = vi.spyOn(catalog, 'getToolByName').mockReturnValue({
                build() {
                    return this.base
                },
                base: {
                    schema: z.object({ name: z.string(), query: z.object({ kind: z.string() }).optional() }),
                    handler: async () => ({ ok: true }),
                },
            } as any)

            const result = (await executor.handleToolCall(
                { name: 'mock-tool', arguments: args },
                makeToolExecutorState([{ name: 'mock-tool' }], { useSingleExec: false })
            )) as any

            expect(result.isError).toBeFalsy()
            expect(result.content[0].text.includes('Ignored input keys')).toBe(reported)
        })
    })
})