:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { closeHub, createHub } from '~/common/utils/db/hub'
import { PostgresUse } from '~/common/utils/db/postgres'
import { createTeam, createTestTeamFixture } from '~/tests/helpers/sql'
import { INVALID_TEAM_ID_KEYS } from '~/tests/helpers/team'
import { Hub } from '~/types'
import { insertEvaluation, insertModelConfiguration, insertProviderKey } from '../_tests/fixtures'
import { Evaluation } from '../types'
import { EvaluationManagerService } from './evaluation-manager.service'
describe('EvaluationManagerService', () => {
jest.setTimeout(2000)
let hub: Hub
let manager: EvaluationManagerService
let evaluations: Evaluation[]
let teamId1: number
let teamId2: number
beforeEach(async () => {
hub = await createHub()
manager = new EvaluationManagerService(hub.postgres, hub.pubSub)
const { organizationId, team } = await createTestTeamFixture(hub.postgres)
teamId1 = team.id
teamId2 = await createTeam(hub.postgres, organizationId)
evaluations = []
evaluations.push(
await insertEvaluation(hub.postgres, teamId1, {
name: 'Test Evaluation team 1',
evaluation_type: 'llm_judge',
evaluation_config: { prompt: 'Test prompt 1' },
output_type: 'boolean',
output_config: {},
enabled: true,
})
)
evaluations.push(
await insertEvaluation(hub.postgres, teamId1, {
name: 'Test Evaluation team 1 - disabled',
evaluation_type: 'llm_judge',
evaluation_config: { prompt: 'Test prompt disabled' },
output_type: 'boolean',
output_config: {},
enabled: false,
})
)
evaluations.push(
await insertEvaluation(hub.postgres, teamId2, {
name: 'Test Evaluation team 2',
evaluation_type: 'llm_judge',
evaluation_config: { prompt: 'Test prompt 2' },
output_type: 'boolean',
output_config: {},
enabled: true,
})
)
})
afterEach(async () => {
await closeHub(hub)
})
it('returns evaluations for a team', async () => {
const items = await manager.getEvaluationsForTeam(teamId1)
expect(items).toHaveLength(1)
expect(items[0].id).toEqual(evaluations[0].id)
expect(items[0].team_id).toEqual(teamId1)
expect(items[0].name).toEqual('Test Evaluation team 1')
})
it('returns evaluations for multiple teams in batch', async () => {
const result = await manager.getEvaluationsForTeams([teamId1, teamId2])
expect(result[teamId1]).toHaveLength(1)
expect(result[teamId2]).toHaveLength(1)
expect(result[teamId1][0].id).toEqual(evaluations[0].id)
expect(result[teamId2][0].id).toEqual(evaluations[2].id)
})
it('returns empty array for teams with no evaluations', async () => {
const nonExistentTeamId = teamId2 + 1
const items = await manager.getEvaluationsForTeam(nonExistentTeamId)
expect(items).toEqual([])
})
it('filters out disabled evaluations', async () => {
const items = await manager.getEvaluationsForTeam(teamId1)
expect(items).toHaveLength(1)
expect(items[0].id).toEqual(evaluations[0].id)
})
it('filters out deleted evaluations', async () => {
// Mark first evaluation as deleted
await hub.postgres.query(
PostgresUse.COMMON_WRITE,
`UPDATE llm_analytics_evaluation SET deleted=true, updated_at = NOW() WHERE id = $1`,
[evaluations[0].id],
'testKey'
)
// This is normally dispatched by django
manager['onEvaluationsReloaded'](teamId1, [evaluations[0].id])
const items = await manager.getEvaluationsForTeam(teamId1)
expect(items).toHaveLength(0)
})
it('caches evaluations and uses cache on subsequent calls', async () => {
// First call
const items1 = await manager.getEvaluationsForTeam(teamId1)
expect(items1).toHaveLength(1)
// Update the database without triggering reload
await hub.postgres.query(
PostgresUse.COMMON_WRITE,
`UPDATE llm_analytics_evaluation SET name='Updated Name', updated_at = NOW() WHERE id = $1`,
[evaluations[0].id],
'testKey'
)
// Second call should still return cached data
const items2 = await manager.getEvaluationsForTeam(teamId1)
expect(items2).toHaveLength(1)
expect(items2[0].name).toEqual('Test Evaluation team 1') // Not updated yet
})
it('reloads evaluations when pubsub message received', async () => {
const itemsBefore = await manager.getEvaluationsForTeam(teamId1)
expect(itemsBefore).toHaveLength(1)
await hub.postgres.query(
PostgresUse.COMMON_WRITE,
`UPDATE llm_analytics_evaluation SET name='Updated Evaluation', updated_at = NOW() WHERE id = $1`,
[evaluations[0].id],
'testKey'
)
manager['onEvaluationsReloaded'](teamId1, [evaluations[0].id])
const itemsAfter = await manager.getEvaluationsForTeam(teamId1)
expect(itemsAfter).toMatchObject([
{
id: evaluations[0].id,
name: 'Updated Evaluation',
},
])
})
it('filters out evaluation when disabled via reload', async () => {
const itemsBefore = await manager.getEvaluationsForTeam(teamId1)
expect(itemsBefore).toHaveLength(1)
await hub.postgres.query(
PostgresUse.COMMON_WRITE,
`UPDATE llm_analytics_evaluation SET enabled=false, updated_at = NOW() WHERE id = $1`,
[evaluations[0].id],
'testKey'
)
manager['onEvaluationsReloaded'](teamId1, [evaluations[0].id])
const itemsAfter = await manager.getEvaluationsForTeam(teamId1)
expect(itemsAfter).toHaveLength(0)
})
it('handles non-existent team IDs gracefully in batch fetch', async () => {
const nonExistentTeamId = teamId2 + 100
const result = await manager.getEvaluationsForTeams([teamId1, nonExistentTeamId, teamId2])
expect(result[teamId1]).toHaveLength(1)
expect(result[nonExistentTeamId]).toEqual([])
expect(result[teamId2]).toHaveLength(1)
})
it.each(INVALID_TEAM_ID_KEYS)(
'returns no evaluations for invalid team id %p without failing the valid team in the same load',
async (key) => {
const invalidTeamId = key as unknown as number
const result = await manager.getEvaluationsForTeams([teamId1, invalidTeamId])
expect(result[teamId1]).toHaveLength(1)
expect(result[invalidTeamId]).toEqual([])
}
)
it('handles evaluations with bytecode errors', async () => {
await insertEvaluation(hub.postgres, teamId1, {
name: 'Evaluation with bytecode error',
evaluation_type: 'llm_judge',
evaluation_config: { prompt: 'Test prompt' },
output_type: 'boolean',
output_config: {},
enabled: true,
conditions: [
{
id: 'cond-1',
rollout_percentage: 100,
properties: [],
bytecode_error: 'Failed to compile',
},
],
})
const items = await manager.getEvaluationsForTeam(teamId1)
expect(items).toHaveLength(2)
const evalWithError = items.find((item) => item.name === 'Evaluation with bytecode error')
expect(evalWithError).toBeDefined()
expect(evalWithError!.conditions[0].bytecode_error).toEqual('Failed to compile')
})
it('returns provider key id from model configuration', async () => {
const providerKey = await insertProviderKey(hub.postgres, teamId1)
const modelConfiguration = await insertModelConfiguration(hub.postgres, teamId1, {
provider_key_id: providerKey.id,
})
await hub.postgres.query(
PostgresUse.COMMON_WRITE,
`UPDATE llm_analytics_evaluation SET model_configuration_id = $1, updated_at = NOW() WHERE id = $2`,
[modelConfiguration.id, evaluations[0].id],
'setEvaluationModelConfiguration'
)
manager['onEvaluationsReloaded'](teamId1, [evaluations[0].id])
const items = await manager.getEvaluationsForTeam(teamId1)
expect(items[0].provider_key_id).toEqual(providerKey.id)
})
})