:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { describe, expect, it } from 'vitest'
import { z } from 'zod'
import { formatInputValidationError, rewrapFlattenedArguments } from '@/tools/exec'
import { GENERATED_TOOL_MAP } from '@/tools/generated'
import { getToolDefinition } from '@/tools/toolDefinitions'
import type { ToolBase, ZodObjectAny } from '@/tools/types'
/** Read from the schemas rather than listed, so a new tool with this shape has to satisfy the contract too. */
function queryWrapperTools(): [string, ToolBase<ZodObjectAny>][] {
const tools: [string, ToolBase<ZodObjectAny>][] = []
for (const [name, factory] of Object.entries(GENERATED_TOOL_MAP)) {
const tool = factory()
let jsonSchema: Record<string, unknown>
try {
jsonSchema = z.toJSONSchema(tool.schema, { io: 'input' }) as Record<string, unknown>
} catch {
continue
}
const required = jsonSchema['required']
const properties = jsonSchema['properties'] as Record<string, { type?: string }> | undefined
if (Array.isArray(required) && required.length === 1 && required[0] === 'query') {
if (properties?.['query']?.type === 'object') {
tools.push([name, tool])
}
}
}
return tools
}
function leadExample(
description: string
): { json: Record<string, unknown>; source: string; offset: number } | undefined {
const match = /```json\n([\s\S]*?)\n```/.exec(description)
if (!match) {
return undefined
}
const source = match[1]!
try {
return { json: JSON.parse(source) as Record<string, unknown>, source, offset: match.index }
} catch {
return undefined
}
}
/**
* The lead example with its window flattened one level further: `date_from` beside the
* other fields rather than inside `dateRange`. This is the shape callers send most, and
* undoing only the outer wrapper leaves it rejected.
*/
function flattenedWindowCall(example: Record<string, unknown>): Record<string, unknown> | undefined {
const query = example['query'] as Record<string, unknown>
const range = query['dateRange']
if (typeof range !== 'object' || range === null || Array.isArray(range)) {
return undefined
}
const { date_from: dateFrom, ...otherBounds } = range as Record<string, unknown>
if (typeof dateFrom !== 'string' || Object.keys(otherBounds).length > 0) {
return undefined
}
const { dateRange: _window, ...fields } = query
return { ...fields, date_from: dateFrom }
}
/** An example below this offset sits under a workflow section, which a caller reaches after it composed the call. */
const LEAD_EXAMPLE_MAX_OFFSET = 700
/** Every description ships on every tools/list, and a longer example also wraps under `oxfmt`. */
const LEAD_EXAMPLE_MAX_LENGTH = 110
describe('tools whose payload sits under a required `query` object', () => {
const tools = queryWrapperTools()
it('finds the wrapper tools to check', () => {
expect(tools.length).toBeGreaterThan(10)
})
it('covers the flattened window on most of them', () => {
const withWindow = tools.filter(([name]) =>
flattenedWindowCall(leadExample(getToolDefinition(name).description)!.json)
)
expect(withWindow.length).toBeGreaterThan(10)
})
describe.each(tools)('%s', (name, tool) => {
const description = getToolDefinition(name).description
const example = leadExample(description)
it('leads with a wrapped example the caller can copy', () => {
expect(example).not.toBeUndefined()
expect(Object.keys(example!.json)).toContain('query')
expect(example!.offset).toBeLessThan(LEAD_EXAMPLE_MAX_OFFSET)
expect(example!.source).not.toContain('\n')
expect(example!.source.length).toBeLessThanOrEqual(LEAD_EXAMPLE_MAX_LENGTH)
})
it('accepts that example', () => {
const parsed = tool.schema.safeParse(example!.json)
expect(parsed.error?.issues ?? []).toEqual([])
})
it('names the nesting mistake when that example is sent flattened', () => {
const flattened = example!.json['query'] as Record<string, unknown>
const rejected = tool.schema.safeParse(flattened, { reportInput: true })
expect(rejected.success).toBe(false)
const message = formatInputValidationError(name, rejected.error!, flattened, tool.schema)
expect(message).toContain('missing required parameter: query')
expect(message).toContain('the fields you sent belong inside it')
expect(message).toContain('resend them as {"query": {')
})
it.each([
['as the caller sent it', (json: Record<string, unknown>) => json['query'] as Record<string, unknown>],
['with its window flattened too', flattenedWindowCall],
])('rewraps the example flattened %s back into the call the caller meant', (_label, flatten) => {
const flattened = flatten(example!.json)
if (!flattened) {
return
}
const rejected = tool.schema.safeParse(flattened, { reportInput: true })
expect(rewrapFlattenedArguments(rejected.error!, flattened, tool.schema)).toEqual(example!.json)
})
})
})