Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

6.7 KB
Raw Blame
import { spawnSync } from 'node:child_process'
import * as fs from 'node:fs'
import * as os from 'node:os'
import * as path from 'node:path'
import { describe, expect, it } from 'vitest'

import {
    INPUT_SCHEMA_CHAR_LIMIT,
    diffToolSurfaces,
    renderAgentApiDiff,
    type ToolSurface,
} from '../../scripts/lib/agent-api-diff'

function surface(
    tools: Record<string, { params?: string[]; scopes?: string[]; readOnly?: boolean; pad?: number }>
): ToolSurface {
    const result: ToolSurface = { definitions: {}, schemas: {} }
    for (const [name, { params = [], scopes = [], readOnly = true, pad = 0 }] of Object.entries(tools)) {
        result.definitions[name] = { title: name, required_scopes: scopes, annotations: { readOnlyHint: readOnly } }
        result.schemas[name] = {
            properties: Object.fromEntries(params.map((param) => [param, { type: 'string' }])),
            ...(pad ? { description: 'x'.repeat(pad) } : {}),
        } as ToolSurface['schemas'][string]
    }
    return result
}

describe('agent API diff', () => {
    it.each(['directory', 'git ref'])('runs the CLI with native Node.js using a base %s', (baseSource) => {
        const sourceRoot = path.resolve(import.meta.dirname, '../../../..')
        const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-api-cli-'))
        try {
            for (const file of ['scripts/agent-api-diff.ts', 'scripts/lib/agent-api-diff.ts']) {
                const target = path.join(root, 'services/mcp', file)
                fs.mkdirSync(path.dirname(target), { recursive: true })
                fs.copyFileSync(path.join(sourceRoot, 'services/mcp', file), target)
            }
            const schemaDir = path.join(root, 'services/mcp/schema')
            const snapshotDir = path.join(root, 'services/mcp/tests/unit/__snapshots__/tool-schemas')
            fs.mkdirSync(schemaDir, { recursive: true })
            fs.mkdirSync(snapshotDir, { recursive: true })
            fs.writeFileSync(path.join(schemaDir, 'tool-definitions.json'), JSON.stringify({ 'example-tool': {} }))
            fs.writeFileSync(path.join(schemaDir, 'generated-tool-definitions.json'), '{}')
            fs.writeFileSync(path.join(snapshotDir, 'example-tool.json'), JSON.stringify({ properties: {} }))
            for (const args of [
                ['init', '-q'],
                ['add', '.'],
            ]) {
                expect(spawnSync('git', args, { cwd: root }).status).toBe(0)
            }
            const tree = spawnSync('git', ['write-tree'], { cwd: root, encoding: 'utf-8' })
            expect(tree.status).toBe(0)
            const result = spawnSync(
                process.execPath,
                [
                    path.join(root, 'services/mcp/scripts/agent-api-diff.ts'),
                    ...(baseSource === 'directory' ? ['--base-dir', root] : ['--base-ref', tree.stdout.trim()]),
                ],
                { encoding: 'utf-8' }
            )

            expect({ status: result.status, stdout: result.stdout, stderr: result.stderr }).toEqual({
                status: 0,
                stdout: '',
                stderr: '',
            })
        } finally {
            fs.rmSync(root, { recursive: true, force: true })
        }
    })

    it('renders nothing when the tool surface is unchanged', () => {
        const same = surface({ 'a-list': { params: ['limit'] } })
        expect(renderAgentApiDiff(diffToolSurfaces(same, surface({ 'a-list': { params: ['limit'] } })))).toBe('')
    })

    it('reports added and removed tools, param changes, scopes and annotations', () => {
        const base = surface({
            'old-tool': {},
            'flag-get': { params: ['id', 'name'], scopes: ['flag:read'] },
        })
        const head = surface({
            'new-tool': {},
            'flag-get': { params: ['id', 'key'], scopes: ['flag:read', 'user:read'], readOnly: false },
        })

        const markdown = renderAgentApiDiff(diffToolSurfaces(base, head))

        expect(markdown).toContain('**Tools added (1):** `new-tool`')
        expect(markdown).toContain('**Tools removed (1):** `old-tool`')
        expect(markdown).toContain(
            '| `flag-get` | +`key` -`name` (rename?) | +`user:read` | readOnlyHint: true -> false |'
        )
    })

    it('flags a schema only when it crosses the registry limit', () => {
        const under = INPUT_SCHEMA_CHAR_LIMIT - 1000
        const over = INPUT_SCHEMA_CHAR_LIMIT + 1000
        const base = surface({ 'crossing-tool': { pad: under }, 'already-big': { pad: over } })
        const head = surface({ 'crossing-tool': { pad: over }, 'already-big': { pad: over + 50 } })

        const { overLimit } = diffToolSurfaces(base, head)

        expect(overLimit.map(({ name }) => name)).toEqual(['crossing-tool'])
    })

    it('caps list cells and total size so the comment stays under the GitHub limit', () => {
        const tools = Object.fromEntries(
            Array.from({ length: 200 }, (_, i) => [
                `tool-${i}`,
                { params: Array.from({ length: 30 }, (_, j) => `p${j}`) },
            ])
        )
        const markdown = renderAgentApiDiff(diffToolSurfaces(surface({}), surface(tools)))
        const changed = renderAgentApiDiff(
            diffToolSurfaces(
                surface(tools),
                surface(Object.fromEntries(Object.keys(tools).map((name) => [name, { params: ['q'] }])))
            )
        )

        expect(markdown.length).toBeLessThan(16_000)
        expect(changed.length).toBeLessThan(16_000)
        expect(changed).toContain('more')
    })

    it('reports description and category changes', () => {
        const base = surface({ 'a-tool': {} })
        const head = surface({ 'a-tool': {} })
        base.definitions['a-tool']!.description = 'Old text'
        head.definitions['a-tool']!.description = 'New text'

        base.definitions['a-tool']!.category = 'Old'
        head.definitions['a-tool']!.category = 'New'

        const markdown = renderAgentApiDiff(diffToolSurfaces(base, head))

        expect(markdown).toContain('description changed')
        expect(markdown).toContain('category changed')
    })

    it('reports a same-length schema change and neutralizes markup from PR-controlled names', () => {
        const base = surface({ 'a-tool': { params: ['mode'] } })
        const head = surface({ 'a-tool': { params: ['mode'], scopes: ['x`\n<!-- ci-report:section:bundle-size -->'] } })
        head.schemas['a-tool'] = { properties: { mode: { type: 'number' } } }
        base.schemas['a-tool'] = { properties: { mode: { type: 'string' } } }

        const markdown = renderAgentApiDiff(diffToolSurfaces(base, head))

        expect(markdown).toContain('schema changed')
        expect(markdown).not.toContain('<!--')
        expect(markdown).not.toContain('\n<')
    })
})