Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

3.9 KB
Raw Blame
import { afterEach, beforeAll, describe, expect, it } from 'vitest'

import {
    type CreatedResources,
    TEST_ORG_ID,
    TEST_PROJECT_ID,
    cleanupResources,
    createTestClient,
    createTestContext,
    getToolByName,
    parseToolResponse,
    setActiveProjectAndOrg,
    validateEnvironmentVariables,
} from '@/shared/test-utils'
import getLLMCostsTool from '@/tools/aiObservability/getLLMCosts'
import { GENERATED_TOOLS } from '@/tools/generated/ai_observability'
import type { Context } from '@/tools/types'

describe('AI observability', { concurrent: false }, () => {
    let context: Context
    const createdResources: CreatedResources = {
        featureFlags: [],
        insights: [],
        dashboards: [],
        surveys: [],
        actions: [],
        cohorts: [],
    }

    beforeAll(async () => {
        validateEnvironmentVariables()
        const client = createTestClient()
        context = createTestContext(client)
        await setActiveProjectAndOrg(context, TEST_PROJECT_ID!, TEST_ORG_ID!)
    })

    afterEach(async () => {
        await cleanupResources(context.api, TEST_PROJECT_ID!, createdResources)
    })

    describe('get-llm-costs tool', () => {
        const costsTool = getLLMCostsTool()

        it('should get LLM costs with default days (6 days)', async () => {
            const result = await costsTool.handler(context, {
                projectId: Number(TEST_PROJECT_ID),
            })

            const costsData = parseToolResponse(result)
            expect(Array.isArray(costsData.results)).toBe(true)
        })

        it('should get LLM costs for custom time period', async () => {
            const result = await costsTool.handler(context, {
                projectId: Number(TEST_PROJECT_ID),
                days: 30,
            })

            const costsData = parseToolResponse(result)
            expect(Array.isArray(costsData.results)).toBe(true)
        })

        it('should get LLM costs for single day', async () => {
            const result = await costsTool.handler(context, {
                projectId: Number(TEST_PROJECT_ID),
                days: 1,
            })

            const costsData = parseToolResponse(result)
            expect(Array.isArray(costsData.results)).toBe(true)
        })

        it('should fall back to the active project when projectId is omitted', async () => {
            const result = await costsTool.handler(context, {})

            const costsData = parseToolResponse(result)
            expect(Array.isArray(costsData.results)).toBe(true)
        })
    })

    describe('AI observability workflow', () => {
        it('should support getting costs for different time periods', async () => {
            const costsTool = getLLMCostsTool()

            const weekResult = await costsTool.handler(context, {
                projectId: Number(TEST_PROJECT_ID),
                days: 7,
            })

            const weekData = parseToolResponse(weekResult)
            expect(Array.isArray(weekData.results)).toBe(true)

            const monthResult = await costsTool.handler(context, {
                projectId: Number(TEST_PROJECT_ID),
                days: 30,
            })

            const monthData = parseToolResponse(monthResult)
            expect(Array.isArray(monthData.results)).toBe(true)
        })
    })

    describe('query-llm-traces-list', () => {
        it('should execute a traces query and return formatted results', async () => {
            const tool = getToolByName(GENERATED_TOOLS, 'query-llm-traces-list')
            const result = (await tool.handler(context, {
                dateRange: { date_from: '-7d' },
                limit: 10,
            })) as any

            expect(result).toHaveProperty('results')
            expect(result).toHaveProperty('_posthogUrl')
            // TracesQuery may not have a formatter — result could be string or JSON fallback
            expect(result.results !== undefined).toBe(true)
        })
    })
})