:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { spawnSync } from 'node:child_process'
import * as fs from 'node:fs'
import * as os from 'node:os'
import * as path from 'node:path'
import { describe, expect, it } from 'vitest'
import {
INPUT_SCHEMA_CHAR_LIMIT,
diffToolSurfaces,
renderAgentApiDiff,
type ToolSurface,
} from '../../scripts/lib/agent-api-diff'
function surface(
tools: Record<string, { params?: string[]; scopes?: string[]; readOnly?: boolean; pad?: number }>
): ToolSurface {
const result: ToolSurface = { definitions: {}, schemas: {} }
for (const [name, { params = [], scopes = [], readOnly = true, pad = 0 }] of Object.entries(tools)) {
result.definitions[name] = { title: name, required_scopes: scopes, annotations: { readOnlyHint: readOnly } }
result.schemas[name] = {
properties: Object.fromEntries(params.map((param) => [param, { type: 'string' }])),
...(pad ? { description: 'x'.repeat(pad) } : {}),
} as ToolSurface['schemas'][string]
}
return result
}
describe('agent API diff', () => {
it.each(['directory', 'git ref'])('runs the CLI with native Node.js using a base %s', (baseSource) => {
const sourceRoot = path.resolve(import.meta.dirname, '../../../..')
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-api-cli-'))
try {
for (const file of ['scripts/agent-api-diff.ts', 'scripts/lib/agent-api-diff.ts']) {
const target = path.join(root, 'services/mcp', file)
fs.mkdirSync(path.dirname(target), { recursive: true })
fs.copyFileSync(path.join(sourceRoot, 'services/mcp', file), target)
}
const schemaDir = path.join(root, 'services/mcp/schema')
const snapshotDir = path.join(root, 'services/mcp/tests/unit/__snapshots__/tool-schemas')
fs.mkdirSync(schemaDir, { recursive: true })
fs.mkdirSync(snapshotDir, { recursive: true })
fs.writeFileSync(path.join(schemaDir, 'tool-definitions.json'), JSON.stringify({ 'example-tool': {} }))
fs.writeFileSync(path.join(schemaDir, 'generated-tool-definitions.json'), '{}')
fs.writeFileSync(path.join(snapshotDir, 'example-tool.json'), JSON.stringify({ properties: {} }))
for (const args of [
['init', '-q'],
['add', '.'],
]) {
expect(spawnSync('git', args, { cwd: root }).status).toBe(0)
}
const tree = spawnSync('git', ['write-tree'], { cwd: root, encoding: 'utf-8' })
expect(tree.status).toBe(0)
const result = spawnSync(
process.execPath,
[
path.join(root, 'services/mcp/scripts/agent-api-diff.ts'),
...(baseSource === 'directory' ? ['--base-dir', root] : ['--base-ref', tree.stdout.trim()]),
],
{ encoding: 'utf-8' }
)
expect({ status: result.status, stdout: result.stdout, stderr: result.stderr }).toEqual({
status: 0,
stdout: '',
stderr: '',
})
} finally {
fs.rmSync(root, { recursive: true, force: true })
}
})
it('renders nothing when the tool surface is unchanged', () => {
const same = surface({ 'a-list': { params: ['limit'] } })
expect(renderAgentApiDiff(diffToolSurfaces(same, surface({ 'a-list': { params: ['limit'] } })))).toBe('')
})
it('reports added and removed tools, param changes, scopes and annotations', () => {
const base = surface({
'old-tool': {},
'flag-get': { params: ['id', 'name'], scopes: ['flag:read'] },
})
const head = surface({
'new-tool': {},
'flag-get': { params: ['id', 'key'], scopes: ['flag:read', 'user:read'], readOnly: false },
})
const markdown = renderAgentApiDiff(diffToolSurfaces(base, head))
expect(markdown).toContain('**Tools added (1):** `new-tool`')
expect(markdown).toContain('**Tools removed (1):** `old-tool`')
expect(markdown).toContain(
'| `flag-get` | +`key` -`name` (rename?) | +`user:read` | readOnlyHint: true -> false |'
)
})
it('flags a schema only when it crosses the registry limit', () => {
const under = INPUT_SCHEMA_CHAR_LIMIT - 1000
const over = INPUT_SCHEMA_CHAR_LIMIT + 1000
const base = surface({ 'crossing-tool': { pad: under }, 'already-big': { pad: over } })
const head = surface({ 'crossing-tool': { pad: over }, 'already-big': { pad: over + 50 } })
const { overLimit } = diffToolSurfaces(base, head)
expect(overLimit.map(({ name }) => name)).toEqual(['crossing-tool'])
})
it('caps list cells and total size so the comment stays under the GitHub limit', () => {
const tools = Object.fromEntries(
Array.from({ length: 200 }, (_, i) => [
`tool-${i}`,
{ params: Array.from({ length: 30 }, (_, j) => `p${j}`) },
])
)
const markdown = renderAgentApiDiff(diffToolSurfaces(surface({}), surface(tools)))
const changed = renderAgentApiDiff(
diffToolSurfaces(
surface(tools),
surface(Object.fromEntries(Object.keys(tools).map((name) => [name, { params: ['q'] }])))
)
)
expect(markdown.length).toBeLessThan(16_000)
expect(changed.length).toBeLessThan(16_000)
expect(changed).toContain('more')
})
it('reports description and category changes', () => {
const base = surface({ 'a-tool': {} })
const head = surface({ 'a-tool': {} })
base.definitions['a-tool']!.description = 'Old text'
head.definitions['a-tool']!.description = 'New text'
base.definitions['a-tool']!.category = 'Old'
head.definitions['a-tool']!.category = 'New'
const markdown = renderAgentApiDiff(diffToolSurfaces(base, head))
expect(markdown).toContain('description changed')
expect(markdown).toContain('category changed')
})
it('reports a same-length schema change and neutralizes markup from PR-controlled names', () => {
const base = surface({ 'a-tool': { params: ['mode'] } })
const head = surface({ 'a-tool': { params: ['mode'], scopes: ['x`\n<!-- ci-report:section:bundle-size -->'] } })
head.schemas['a-tool'] = { properties: { mode: { type: 'number' } } }
base.schemas['a-tool'] = { properties: { mode: { type: 'string' } } }
const markdown = renderAgentApiDiff(diffToolSurfaces(base, head))
expect(markdown).toContain('schema changed')
expect(markdown).not.toContain('<!--')
expect(markdown).not.toContain('\n<')
})
})