Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

7.9 KB
Raw Blame
import http from 'node:http'
import { gunzipSync } from 'node:zlib'
import { afterAll, beforeAll, describe, expect, it } from 'vitest'

import { toolFromPreBuilt } from '../shared/test-utils'

/**
 * End-to-end proof that a real skill read reaches PostHog carrying the skill's name.
 *
 * Only the destination is substituted: a local HTTP sink stands in for ingestion so
 * the outbound payload can be read. Everything upstream is real — the real
 * ToolExecutor, the real `exec` dispatcher, the real `skill-get` handler fetching a
 * real skill over HTTPS, and the real analytics client serializing the event.
 *
 *   POSTHOG_PERSONAL_API_KEY=... npx vitest run \
 *     tests/integration/skill-analytics.integration.test.ts \
 *     --config vitest.integration.config.mts
 *
 * Run by hand, not by CI. It reads `POSTHOG_PERSONAL_API_KEY` rather than the
 * `TEST_`-prefixed variables the CI integration job supplies, so it always skips
 * there. That is deliberate: it names one skill on one real instance, which is a
 * fact about that project rather than about this code, so asserting it in CI would
 * fail against the e2e stack and add a network dependency to a suite that has none.
 * The wiring itself is covered by the unit and hono suites, which do run in CI.
 */

const API_TOKEN = process.env.POSTHOG_PERSONAL_API_KEY
const API_HOST = process.env.POSTHOG_API_HOST ?? 'https://us.posthog.com'
const PROJECT_ID = process.env.POSTHOG_PROJECT_ID ?? '2'
// A skill that exists in the target project. Override when running elsewhere.
const SKILL = process.env.E2E_SKILL_NAME ?? 'conductor'

const captured: any[] = []
let sink: http.Server
let sinkPort: number

describe.skipIf(!API_TOKEN)('skill read reaches PostHog with $mcp_skill_name', () => {
    beforeAll(async () => {
        sink = http.createServer((req, res) => {
            const chunks: Buffer[] = []
            req.on('data', (c: Buffer) => chunks.push(c))
            req.on('end', () => {
                try {
                    const raw = Buffer.concat(chunks)
                    // posthog-node gzips the batch before POSTing it to /batch/.
                    const json = req.headers['content-encoding'] === 'gzip' ? gunzipSync(raw) : raw
                    const body = JSON.parse(json.toString())
                    for (const item of body.batch ?? [body]) {
                        captured.push(item)
                    }
                } catch {
                    // non-JSON probe; ignore
                }
                res.writeHead(200, { 'content-type': 'application/json' })
                res.end('{"status":1}')
            })
        })
        await new Promise<void>((resolve) => sink.listen(0, '127.0.0.1', resolve))
        sinkPort = (sink.address() as { port: number }).port

        // Set before the analytics client is imported — it is a module-level singleton.
        process.env.POSTHOG_ANALYTICS_API_KEY = 'phc_e2e_local_sink'
        process.env.POSTHOG_ANALYTICS_HOST = `http://127.0.0.1:${sinkPort}`
    })

    afterAll(async () => {
        await new Promise<void>((resolve) => sink?.close(() => resolve()))
    })

    it('fetches the real skill and emits an event naming it', async () => {
        const [{ ApiClient }, { ToolCatalog }, { ToolExecutor }, { InstructionsBuilder }, { getPostHogClient }] =
            await Promise.all([
                import('@/api/client'),
                import('@/hono/tool-catalog'),
                import('@/hono/tool-executor'),
                import('@/hono/instructions'),
                import('@/lib/posthog/client'),
            ])

        const api = new ApiClient({ apiToken: API_TOKEN!, baseUrl: API_HOST })
        const catalog = new ToolCatalog()
        await catalog.warmup()
        const executor = new ToolExecutor(catalog, new InstructionsBuilder(''))

        const tools = catalog
            .getPreBuiltEntries()
            .map((entry) => toolFromPreBuilt(catalog.getToolByName(entry.name)!, entry))

        const state: any = {
            reqCtx: {
                cache: { get: async () => undefined, set: async () => undefined },
                safelyGetAnalyticsContext: async () => undefined,
                trackEvent: () => {},
                trackContextSwitchEvent: () => {},
                getSessionUuid: async () => 'e2e-session',
                getEffectiveSessionUuid: async () => 'e2e-session',
            },
            context: {
                api,
                cache: {},
                env: {},
                stateManager: { getProjectId: async () => PROJECT_ID },
                sessionManager: {},
                getDistinctId: () => 'e2e-distinct-id',
                trackEvent: () => {},
            },
            useSingleExec: true,
            toolFeatureFlags: undefined,
            apiKeyScopes: [],
            oauthClientId: undefined,
            clientProfile: {
                capabilities: { supportsInstructions: true },
                isCliModeEnabled: () => true,
                isInlineExecUiHost: () => false,
                isClaudeChatHost: () => false,
                isAnthropicConnector: () => false,
            },
            requestContext: {
                authMethod: 'personal_api_key',
                sessionId: 'e2e-session',
                mcpClientName: 'e2e-harness',
                mcpClientVersion: '1.0',
                mcpProtocolVersion: '2025-03-26',
                transport: 'streamable-http',
            },
            sessionContext: null,
            allTools: tools,
            scopeGatedTools: [],
            findScopeGatedTool: () => undefined,
            supportsStepUp: false,
            flagGatedTools: [],
            gatewayToolsEnabled: false,
            distinctId: 'e2e-distinct-id',
            renderUiEnabled: false,
        }

        // Exactly the shape an agent sends: single-exec mode, skill name inside the command.
        const command = `call skill-get {"skill_name":"${SKILL}","body_offset":0}`
        console.info(`\n→ MCP tools/call   exec   ${command}`)

        const result: any = await executor.handleToolCall({ name: 'exec', arguments: { command } }, state)
        const text: string = result?.content?.[0]?.text ?? ''

        console.info(`← ${API_HOST} returned ${text.length} chars, isError=${result?.isError ?? false}`)

        // `trackToolCall` is fired with `void` and awaits async context lookups, so the
        // capture lands after the handler returns. Let it settle before flushing.
        await new Promise((r) => setTimeout(r, 1500))
        await getPostHogClient().flush()
        await new Promise((r) => setTimeout(r, 1000))
        console.info(`  sink received ${captured.length} event(s): ${captured.map((e) => e.event).join(', ')}`)

        const toolCall = captured.find((e) => e.event === '$mcp_tool_call')
        const props = toolCall?.properties ?? {}

        console.info('\n── $mcp_tool_call as PostHog ingestion would receive it ──')
        console.info(
            JSON.stringify(
                Object.fromEntries(
                    Object.entries(props).filter(([k]) =>
                        [
                            '$mcp_skill_name',
                            '$mcp_skill_body_offset',
                            '$mcp_tool_name',
                            '$mcp_exec_verb',
                            '$mcp_exec_target_tool',
                            '$mcp_is_error',
                        ].includes(k)
                    )
                ),
                null,
                2
            )
        )
        console.info(`\n✓ $mcp_skill_name = ${JSON.stringify(props.$mcp_skill_name)}\n`)

        // The skill really came back from the API, not from a fixture or an error path.
        expect(result?.isError ?? false).toBe(false)
        expect(text.length).toBeGreaterThan(500)

        expect(toolCall).not.toBeUndefined()
        expect(props.$mcp_tool_name).toBe('skill-get')
        expect(props.$mcp_skill_name).toBe(SKILL)
        expect(props.$mcp_skill_body_offset).toBe(0)
    })
})