Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

9.7 KB
Raw Blame
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { z } from 'zod'

import { ToolCatalog, toMcpInputSchema } from '@/hono/tool-catalog'
import type { ToolBase, ZodObjectAny } from '@/tools/types'

type FakeDefinition = {
    title: string
    description: string
    feature: string
    category: string
    required_scopes: string[]
    annotations: {
        destructiveHint: boolean
        idempotentHint: boolean
        openWorldHint: boolean
        readOnlyHint: boolean
    }
    [key: string]: unknown
}

function makeToolBase(name: string, overrides?: Partial<ToolBase<ZodObjectAny>>): ToolBase<ZodObjectAny> {
    return {
        name,
        schema: z.object({}),
        handler: vi.fn(),
        ...overrides,
    }
}

const fakeDef = (overrides?: Record<string, unknown>): FakeDefinition => ({
    title: 'Title',
    description: 'Desc',
    feature: 'general',
    category: 'general',
    required_scopes: [] as string[],
    annotations: {
        destructiveHint: false,
        idempotentHint: false,
        openWorldHint: false,
        readOnlyHint: false,
    },
    ...overrides,
})

vi.mock('@/tools', () => ({
    TOOL_MAP: {
        'tool-a': () => makeToolBase('tool-a'),
        'tool-b': () => makeToolBase('tool-b'),
        'update-feature-flag': () =>
            makeToolBase('update-feature-flag', {
                schema: z.object({ source: z.literal('handwritten') }),
            }),
    },
}))

vi.mock('@/tools/generated', () => ({
    GENERATED_TOOL_MAP: {
        'gen-tool-c': () => makeToolBase('gen-tool-c'),
        'update-feature-flag': () =>
            makeToolBase('update-feature-flag', {
                schema: z.object({ source: z.literal('generated') }),
            }),
    },
}))

const DEFINITIONS: Record<string, FakeDefinition> = {
    'tool-a': fakeDef({ required_scopes: ['project:read'] }),
    'tool-b': fakeDef({ feature: 'insights', annotations: { ...fakeDef().annotations, readOnlyHint: true } }),
    'gen-tool-c': fakeDef({ required_scopes: ['action:write'] }),
    'update-feature-flag': fakeDef({ required_scopes: ['feature_flag:write'] }),
}

vi.mock('@/tools/toolDefinitions', () => ({
    getToolDefinitions: () => DEFINITIONS,
    getToolDefinition: (name: string) => {
        const def = DEFINITIONS[name]
        if (!def) {
            throw new Error(`Tool definition not found for: ${name}`)
        }
        return def
    },
    getToolsForFeatures: (options?: {
        features?: string[]
        tools?: string[]
        readOnly?: boolean
        aiConsentGiven?: boolean
        featureFlags?: Record<string, boolean>
    }) => {
        let names = Object.keys(DEFINITIONS)
        if (options?.features?.length) {
            names = names.filter((n) => {
                const def = DEFINITIONS[n]
                return def && options.features!.includes(def.feature as string)
            })
        }
        if (options?.readOnly) {
            names = names.filter((n) => {
                const def = DEFINITIONS[n]
                return def?.annotations.readOnlyHint === true
            })
        }
        return names
    },
}))

describe('ToolCatalog', () => {
    let catalog: ToolCatalog

    beforeEach(async () => {
        catalog = new ToolCatalog()
    })

    describe('toMcpInputSchema', () => {
        // GitHub Copilot Chat rejects tools whose inputSchema has a root union
        // keyword ("object has unsupported top-level schema keyword 'anyOf'"),
        // so polymorphic request bodies must advertise as one flat object.
        it('flattens a top-level union of object variants into a single object schema', () => {
            const union = z.union([
                z.object({
                    model: z.enum(['events']),
                    file: z.object({ format: z.string() }),
                    include: z.array(z.string()).optional(),
                }),
                z.object({
                    model: z.enum(['persons']),
                    file: z.object({ format: z.string() }),
                }),
            ]) as unknown as ZodObjectAny

            const result = toMcpInputSchema(union) as Record<string, unknown>

            expect(result.anyOf).toBeUndefined()
            expect(result.oneOf).toBeUndefined()
            expect(result.type).toBe('object')
            const properties = result.properties as Record<string, Record<string, unknown>>
            expect(properties.model!.enum).toEqual(['events', 'persons'])
            expect(Object.keys(properties)).toEqual(['model', 'file', 'include'])
            // `include` only exists on one variant, so it must not be required.
            expect(result.required).toEqual(['model', 'file'])
        })

        it('keeps a root type of object when union variants are not all objects', () => {
            const union = z.union([z.object({ a: z.string() }), z.string()]) as unknown as ZodObjectAny

            const result = toMcpInputSchema(union) as Record<string, unknown>

            expect(result.type).toBe('object')
        })

        it('returns plain data with no handle back to the zod instance', () => {
            // zod puts a non-enumerable `~standard` handle on its JSON Schema output. Kept on a
            // catalog entry, it would pin every built schema for the life of the process.
            const result = toMcpInputSchema(z.object({ a: z.string() }))

            expect('~standard' in result).toBe(false)
        })
    })

    describe('warmup', () => {
        it('should import tool modules and build the pre-computed map', async () => {
            expect(catalog.warmedUp).toBe(false)
            await catalog.warmup()
            expect(catalog.warmedUp).toBe(true)
        })

        it('should be idempotent', async () => {
            await catalog.warmup()
            await catalog.warmup()
            expect(catalog.warmedUp).toBe(true)
        })

        it('builds a fresh tool on every lookup instead of keeping one', async () => {
            await catalog.warmup()

            const preBuilt = catalog.getToolByName('gen-tool-c')!

            expect(preBuilt.build().schema).not.toBe(preBuilt.build().schema)
        })
    })

    describe('getFilteredTools', () => {
        beforeEach(async () => {
            await catalog.warmup()
        })

        it('should return all tools when no filters applied', () => {
            const tools = catalog.getFilteredTools({
                scopes: ['project:read', 'action:write', 'feature_flag:write'],
            })
            const names = tools.map((t) => t.name).sort()
            expect(names).toEqual(['gen-tool-c', 'tool-a', 'tool-b', 'update-feature-flag'])
        })

        it('should prefer hand-written factory when name collides with generated', () => {
            const tools = catalog.getFilteredTools({ scopes: ['feature_flag:write'] })
            const update = tools.find((t) => t.name === 'update-feature-flag')
            expect(update).toBeTruthy()
            // Schema is from the TOOL_MAP mock (handwritten), not GENERATED_TOOL_MAP.
            expect(update!.schema.safeParse({ source: 'handwritten' }).success).toBe(true)
            expect(update!.schema.safeParse({ source: 'generated' }).success).toBe(false)
        })

        it('should exclude tools by name', () => {
            const tools = catalog.getFilteredTools({
                scopes: ['project:read', 'action:write'],
                excludeTools: ['tool-b'],
            })
            const names = tools.map((t) => t.name).sort()
            expect(names).toEqual(['gen-tool-c', 'tool-a'])
        })

        it('should filter by scopes', () => {
            const tools = catalog.getFilteredTools({ scopes: ['project:read'] })
            const names = tools.map((t) => t.name).sort()
            expect(names).toEqual(['tool-a', 'tool-b'])
        })

        it('should exclude tools when scopes are missing', () => {
            const tools = catalog.getFilteredTools({ scopes: [] })
            const names = tools.map((t) => t.name)
            expect(names).toEqual(['tool-b'])
        })

        it('should filter by readOnly when passed through to getToolsForFeatures', () => {
            const tools = catalog.getFilteredTools({
                scopes: ['project:read', 'action:write'],
                readOnly: true,
            })
            const names = tools.map((t) => t.name)
            expect(names).toEqual(['tool-b'])
        })

        it('should filter by features when passed through to getToolsForFeatures', () => {
            const tools = catalog.getFilteredTools({
                scopes: ['project:read'],
                features: ['insights'],
            })
            const names = tools.map((t) => t.name)
            expect(names).toEqual(['tool-b'])
        })

        it('should merge title and description from definitions onto the tool', () => {
            const tools = catalog.getFilteredTools({ scopes: ['project:read'] })
            const toolA = tools.find((t) => t.name === 'tool-a')
            expect(toolA).toBeTruthy()
            expect(toolA!.title).toBe('Title')
            expect(toolA!.description).toBe('Desc')
        })

        it('should attach scopes and annotations from definitions', () => {
            const tools = catalog.getFilteredTools({ scopes: ['project:read'] })
            const toolA = tools.find((t) => t.name === 'tool-a')
            expect(toolA!.scopes).toEqual(['project:read'])
            expect(toolA!.annotations).toEqual(expect.objectContaining({ readOnlyHint: false }))
        })

        it('should return empty array when catalog is not warmed up', () => {
            const coldCatalog = new ToolCatalog()
            const tools = coldCatalog.getFilteredTools({ scopes: [] })
            expect(tools).toEqual([])
        })
    })
})