Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

7.0 KB
Raw Blame
import { describe, expect, it } from 'vitest'
import { z } from 'zod'

import { InstructionsBuilder } from '@/hono/instructions'
import type { ResolvedState } from '@/hono/request-state-resolver'
import { MCPClientProfile } from '@/lib/client-detection'
import { createExecTool } from '@/tools/exec'
import { getToolDefinitions } from '@/tools/toolDefinitions'
import type { Context, Tool, ZodObjectAny } from '@/tools/types'

import { makeToolExecutorState } from '../shared/test-utils'

const CATALOG_TOOL_NAMES = Object.keys(getToolDefinitions())
const CATALOG = new Set(CATALOG_TOOL_NAMES)

/** Tools named in prose that predates any availability gate. Each one is still a
 *  live instance of the same defect, kept as a ratchet rather than fixed here:
 *  `analysis-artifacts` and `url-patterns` name dashboard and link tools, the
 *  metric-discovery sections name the data-catalog trio, and `examples`,
 *  `retrieving-data`, `cli-data-discovery` and `schema-workflow` name query, SQL,
 *  schema and skill tools. `cli-rendering` names a tool as an example of one
 *  `render-ui` can render, and `cli-syntax` names one as an example of a namespaced
 *  reference. Remove a name once its section gates the mention. */
const UNGATED_BY_DESIGN = new Set([
    'actions-get-all',
    'agent-feedback',
    'dashboard-create',
    'dashboard-get',
    'dashboard-widgets-batch-add',
    'dashboards-get-all',
    'data-catalog-metric-run',
    'execute-sql',
    'experiment-get',
    'feature-flag-get-all',
    'generate-app-url',
    'insight-create',
    'insight-get',
    'insights-list',
    'metric-describe',
    'metric-list',
    'query-funnel',
    'query-lifecycle',
    'query-llm-traces-list',
    'query-paths',
    'query-retention',
    'query-stickiness',
    'query-trends',
    'read-data-schema',
    'skill-get',
])

/** Backticked kebab-case names the guidance uses, which is the shape every tool
 *  name has. Single-word tokens (`series`, `call`) are prose and cannot be a tool. */
function backtickedNames(text: string): string[] {
    const named = new Set<string>()
    for (const match of text.matchAll(/`([a-z][a-z0-9]*(?:-[a-z0-9]+)+)`/g)) {
        const token = match[1]
        if (token) {
            named.add(token)
        }
    }
    return [...named].sort()
}

/** Unlike `backtickedNames`, this scans inside the span, so a name still counts
 *  when the span holds a whole invocation (`call docs-search <json_input>`). URL
 *  slugs that share the shape drop out against the catalog. */
function namedCatalogTools(text: string): string[] {
    const named = new Set<string>()
    for (const span of text.matchAll(/`([^`\n]+)`/g)) {
        for (const match of (span[1] ?? '').matchAll(/[a-z][a-z0-9]*(?:-[a-z0-9]+)+/g)) {
            if (CATALOG.has(match[0])) {
                named.add(match[0])
            }
        }
    }
    return [...named].sort()
}

/** Kebab-case names that are deliberately not exec commands: a built-in skill
 *  loaded by name, and the render-ui tool, which MCP Apps hosts get as its own
 *  MCP tool rather than through the exec catalog. */
const NOT_EXEC_COMMANDS = new Set(['querying-posthog-data', 'render-ui'])

function advertisedCommands(text: string): string[] {
    return namedCatalogTools(text)
}

/** Both surfaces an agent reads before its first call, plus the separate
 *  reference Claude web/desktop gets. */
function execGuidance(toolNames: string[]): string {
    const builder = new InstructionsBuilder('some guidelines')
    const state = (vendorClient?: string, renderUiEnabled = false): ResolvedState =>
        makeToolExecutorState(
            toolNames.map((name) => ({ name })),
            { clientProfile: new MCPClientProfile(vendorClient ? { vendorClient } : {}), renderUiEnabled }
        )
    return [
        builder.buildExecToolDescription(state()),
        builder.buildExecCommandReference(state()),
        builder.buildExecCommandReference(state('ClaudeAI')),
        builder.buildExecCommandReference(state(undefined, true)),
    ].join('\n\n')
}

/** A command the guidance names is only safe if the guidance stops naming it once
 *  the connection loses the tool — otherwise the agent is sent to a name `search`
 *  and `call` cannot resolve. */
function namesToolWhenWithheld(name: string): boolean {
    const available = CATALOG_TOOL_NAMES.filter((tool) => tool !== name)
    return advertisedCommands(execGuidance(available)).includes(name)
}

const CATALOG_TOOLS: Tool<ZodObjectAny>[] = CATALOG_TOOL_NAMES.map((name) => ({
    name,
    title: name,
    description: name,
    schema: z.object({}),
    scopes: [],
    annotations: { destructiveHint: false, idempotentHint: true, openWorldHint: false, readOnlyHint: true },
    handler: async () => ({}),
}))

async function replyToBareCommand(name: string): Promise<string> {
    const exec = createExecTool(CATALOG_TOOLS, {} as Context, 'description', 'reference', undefined)
    return exec.handler({} as Context, { command: `${name} {}` }).then(
        (result) => String(result),
        (error: Error) => error.message
    )
}

describe('exec guidance advertises only resolvable commands', () => {
    const advertised = advertisedCommands(execGuidance(CATALOG_TOOL_NAMES))
    const gated = advertised.filter((name) => !UNGATED_BY_DESIGN.has(name))

    it('gates every advertised command that is not a known exemption', () => {
        expect(gated.filter(namesToolWhenWithheld)).toEqual([])
    })

    it('keeps the exemption list free of names that are now gated', () => {
        const stale = [...UNGATED_BY_DESIGN].filter((name) => !namesToolWhenWithheld(name)).sort()
        expect(stale).toEqual([])
    })

    it('names no command that the catalog cannot resolve', () => {
        const unresolvable = backtickedNames(execGuidance(CATALOG_TOOL_NAMES)).filter(
            (name) => !CATALOG.has(name) && !NOT_EXEC_COMMANDS.has(name)
        )
        expect(unresolvable).toEqual([])
    })

    it('names docs-search when the connection holds it', () => {
        expect(advertised).toContain('docs-search')
    })

    // Resolvable is not the same as reachable: an agent reading a CLI contract types
    // an advertised name as a command, so the reply must carry the form that works.
    it('routes every advertised command to the invocation the dispatcher accepts', async () => {
        const deadEnds: string[] = []
        for (const name of advertised) {
            const reply = await replyToBareCommand(name)
            if (!reply.includes(`call ${name}`)) {
                deadEnds.push(name)
            }
        }
        expect(deadEnds).toEqual([])
    })
})

describe('exec tool annotations on tools/list', () => {
    it('advertises every hint and does not mark exec as destructive', () => {
        const entry = new InstructionsBuilder('some guidelines').buildExecToolEntry(makeToolExecutorState([]))
        expect(entry.annotations).toEqual({
            destructiveHint: false,
            idempotentHint: false,
            openWorldHint: true,
            readOnlyHint: false,
        })
    })
})