:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { describe, expect, it } from 'vitest'
import { z } from 'zod'
import { createExecTool } from '@/tools/exec'
import { GENERATED_TOOL_MAP } from '@/tools/generated'
import { getToolDefinitions } from '@/tools/toolDefinitions'
import type { Context, Tool, ToolBase, ZodObjectAny } from '@/tools/types'
/**
* The mirror of `nested-query-input-example.test.ts`: a query wrapper takes the
* query fields as its own top-level parameters, so an example written as a query
* document teaches a call the tool rejects.
*
* Every documented example is checked through both routes a caller reaches the
* tool by: the schema directly, and the `exec` CLI behind `call <tool> <json>`.
*/
const TOP_LEVEL_QUERY_TOOLS = [
'query-trends',
'query-funnel',
'query-retention',
'query-stickiness',
'query-paths',
'query-lifecycle',
] as const
const definitions = getToolDefinitions()
/** Every fenced ```json block in a description, parsed; unparseable blocks are
* skipped because a description may show a fragment rather than a whole call. */
function jsonExamples(description: string): unknown[] {
const examples: unknown[] = []
for (const match of description.matchAll(/```json\n([\s\S]*?)```/g)) {
try {
examples.push(JSON.parse(match[1]!))
} catch {
continue
}
}
return examples
}
/** Every field the example sets that the parsed input no longer carries. */
function droppedFields(sent: unknown, accepted: unknown, trail = ''): string[] {
if (Array.isArray(sent)) {
return Array.isArray(accepted)
? sent.flatMap((item, index) => droppedFields(item, accepted[index], `${trail}[${index}]`))
: [trail]
}
if (typeof sent !== 'object' || sent === null) {
return []
}
if (typeof accepted !== 'object' || accepted === null) {
return [trail]
}
return Object.entries(sent).flatMap(([key, value]) => {
const path = trail ? `${trail}.${key}` : key
const held = accepted as Record<string, unknown>
return key in held ? droppedFields(value, held[key], path) : [path]
})
}
const mockContext = { getDistinctId: async () => 'test-distinct-id' } as unknown as Context
/** The generated tool with its handler replaced, so a `call` exercises the
* validation gate without reaching the API. */
function asExecutableTool(name: string, base: ToolBase<ZodObjectAny>): Tool<ZodObjectAny> {
return {
...base,
title: name,
description: definitions[name]?.description ?? '',
scopes: [],
annotations: { destructiveHint: false, idempotentHint: true, openWorldHint: false, readOnlyHint: true },
handler: async () => ({ results: [] }),
}
}
describe('tools that take their query fields at the top level', () => {
it.each(TOP_LEVEL_QUERY_TOOLS)('%s takes no nested query parameter', (name) => {
// Guards the premise: were one of these to grow a `query` wrapper, its
// examples would have to nest, and the cases below would be backwards.
const schema = z.toJSONSchema(GENERATED_TOOL_MAP[name]!().schema, { io: 'input' }) as Record<string, unknown>
const properties = schema['properties'] as Record<string, unknown> | undefined
expect(properties && 'query' in properties).toBeFalsy()
})
it.each(TOP_LEVEL_QUERY_TOOLS)('%s documents examples its own schema accepts', (name) => {
const tool = GENERATED_TOOL_MAP[name]!()
const examples = jsonExamples(definitions[name]?.description ?? '')
expect(examples.length, `${name} needs at least one \`\`\`json example`).toBeGreaterThan(0)
for (const example of examples) {
const result = tool.schema.safeParse(example)
expect(result.success, `${name} documents an example it rejects: ${JSON.stringify(result)}`).toBe(true)
}
})
it.each(TOP_LEVEL_QUERY_TOOLS)('%s accepts its documented examples through the exec CLI', async (name) => {
const exec = createExecTool(
[asExecutableTool(name, GENERATED_TOOL_MAP[name]!())],
mockContext,
'test description',
'test command reference',
undefined
)
for (const example of jsonExamples(definitions[name]?.description ?? '')) {
const output = await exec
.handler(mockContext, { command: `call ${name} ${JSON.stringify(example)}` })
.catch((error: Error) => `rejected: ${error.message}`)
expect(String(output)).not.toContain('Invalid input')
}
})
it.each(TOP_LEVEL_QUERY_TOOLS)(
'%s runs its documented example nested under `query` through the exec CLI',
async (name) => {
// Callers keep sending the natural `{query: {...}}` shape whatever the description
// says, so the executor lifts it to the top level instead of rejecting it.
const received: unknown[] = []
const tool = asExecutableTool(name, GENERATED_TOOL_MAP[name]!())
const exec = createExecTool(
[{ ...tool, handler: async (_context, params) => (received.push(params), { results: [] }) }],
mockContext,
'test description',
'test command reference',
undefined
)
const example = jsonExamples(definitions[name]?.description ?? '')[0] as Record<string, unknown>
const output = await exec.handler(mockContext, {
command: `call ${name} ${JSON.stringify({ query: example })}`,
})
expect(String(output)).not.toContain('Invalid input')
expect(received).toEqual([tool.schema.parse(example)])
}
)
// Parsing alone does not prove an example works: these schemas drop fields
// they do not declare, so an example naming one is accepted and then quietly
// loses it. Scoped to trends — `query-funnel` documents `name` on a grouped
// step's inner nodes, and its schema drops that.
it('query-trends documents examples that keep every field they set', () => {
const tool = GENERATED_TOOL_MAP['query-trends']!()
for (const example of jsonExamples(definitions['query-trends']?.description ?? '')) {
const result = tool.schema.safeParse(example)
expect(result.success).toBe(true)
expect(droppedFields(example, result.data)).toEqual([])
}
})
it('documents both a single-series and a multi-series trends example', () => {
const series = jsonExamples(definitions['query-trends']?.description ?? '')
.filter((example): example is { series: unknown[] } => Array.isArray((example as any)?.series))
.map((example) => example.series.length)
expect(series).toContain(1)
expect(series.some((count) => count > 1)).toBe(true)
})
})