Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
fix(eval): offline evaluations fail after restart — pass databaseConfigId to defaults loader (#1592)
Aftab8a289e638819
diff --git a/src/client/src/__tests__/lib/platform/evaluation/evaluation-config.test.ts b/src/client/src/__tests__/lib/platform/evaluation/evaluation-config.test.tsindex 239184eb..05ec9ba0 100644--- a/src/client/src/__tests__/lib/platform/evaluation/evaluation-config.test.ts+++ b/src/client/src/__tests__/lib/platform/evaluation/evaluation-config.test.ts@@ -220,6 +220,24 @@ describe('getEvaluationConfig', () => {expect(result.evaluationTypes).toBeDefined();expect(result.evaluationTypes!.length).toBeGreaterThan(0);});++ it('loads default prompts for the database config it was given', async () => {+ (asaw as jest.Mock).mockResolvedValueOnce([null, makeMockEvalConfig()]);++ await getEvaluationConfig({ id: 'db-9' } as any);++ expect(getEvaluationTypeDefaultPrompts).toHaveBeenCalledWith('db-9');+ });++ it('loads default prompts for the session database config', async () => {+ (asaw as jest.Mock)+ .mockResolvedValueOnce([null, { id: 'db-9' }])+ .mockResolvedValueOnce([null, makeMockEvalConfig()]);++ await getEvaluationConfig();++ expect(getEvaluationTypeDefaultPrompts).toHaveBeenCalledWith('db-9');+ });});// ---------------------------------------------------------------------------@@ -558,6 +576,15 @@ describe('getEvaluationConfigById', () => {expect(result.evaluationTypes).toBeDefined();expect(result.evaluationTypes!.length).toBeGreaterThan(0);});++ it('loads default prompts for the config row databaseConfigId', async () => {+ const config = { ...makeMockEvalConfig(), databaseConfigId: 'db-9' };+ (asaw as jest.Mock).mockResolvedValueOnce([null, config]);++ await getEvaluationConfigById('eval-cfg-1');++ expect(getEvaluationTypeDefaultPrompts).toHaveBeenCalledWith('db-9');+ });});// ---------------------------------------------------------------------------@@ -609,6 +636,15 @@ describe('getEvaluationConfigByDbConfigId', () => {expect(result.secret).toEqual(secretWithValue);});+ it('loads default prompts for the requested databaseConfigId', async () => {+ (asaw as jest.Mock).mockResolvedValueOnce([null, makeMockEvalConfig()]);+ (getSecretById as jest.Mock).mockResolvedValue({ data: [mockSecret] });++ await getEvaluationConfigByDbConfigId('db-9');++ expect(getEvaluationTypeDefaultPrompts).toHaveBeenCalledWith('db-9');+ });+it('defaults meta to an empty object when config.meta is undefined', async () => {const configWithoutMeta = { ...makeMockEvalConfig(), meta: undefined };(asaw as jest.Mock).mockResolvedValueOnce([null, configWithoutMeta]);diff --git a/src/client/src/__tests__/lib/platform/evaluation/evaluation-type-defaults.test.ts b/src/client/src/__tests__/lib/platform/evaluation/evaluation-type-defaults.test.tsindex 56170c5e..3ea47307 100644--- a/src/client/src/__tests__/lib/platform/evaluation/evaluation-type-defaults.test.ts+++ b/src/client/src/__tests__/lib/platform/evaluation/evaluation-type-defaults.test.ts@@ -1,7 +1,8 @@-// Module-level cache (CACHE + cacheLoaded) means we must isolate modules per test.+// The prompt cache is module-level, so each test needs a fresh module registry.// jest.isolateModulesAsync gives each test a fresh module registry.jest.mock('@/lib/platform/common', () => ({ dataCollector: jest.fn() }));+jest.mock('@/utils/log', () => ({ consoleLog: jest.fn() }));jest.mock('@/lib/platform/evaluation/table-details', () => ({OPENLIT_EVALUATION_TYPE_DEFAULTS_TABLE_NAME: 'openlit_evaluation_type_defaults',}));@@ -91,6 +92,119 @@ describe('getEvaluationTypeDefaultPrompts', () => {expect(dataCollector).toHaveBeenCalledTimes(1);});});++ it('passes databaseConfigId through to dataCollector', async () => {+ await jest.isolateModulesAsync(async () => {+ const { dataCollector } = require('@/lib/platform/common') as { dataCollector: jest.Mock };+ jest.clearAllMocks();+ dataCollector.mockResolvedValue({+ data: [{ id: 'hallucination', default_prompt: 'prompt' }],+ err: null,+ });+ const mod = await import('@/lib/platform/evaluation/evaluation-type-defaults');+ await mod.getEvaluationTypeDefaultPrompts('db-1');+ expect(dataCollector).toHaveBeenCalledWith(+ expect.objectContaining({+ query: expect.stringContaining('openlit_evaluation_type_defaults'),+ }),+ 'query',+ 'db-1'+ );+ });+ });++ it('keeps a separate cache entry per databaseConfigId', async () => {+ await jest.isolateModulesAsync(async () => {+ const { dataCollector } = require('@/lib/platform/common') as { dataCollector: jest.Mock };+ jest.clearAllMocks();+ dataCollector.mockResolvedValue({+ data: [{ id: 'hallucination', default_prompt: 'prompt' }],+ err: null,+ });+ const mod = await import('@/lib/platform/evaluation/evaluation-type-defaults');+ await mod.getEvaluationTypeDefaultPrompts('db-1');+ await mod.getEvaluationTypeDefaultPrompts('db-1');+ await mod.getEvaluationTypeDefaultPrompts('db-2');+ expect(dataCollector).toHaveBeenCalledTimes(2);+ expect(dataCollector).toHaveBeenNthCalledWith(+ 1,+ expect.any(Object),+ 'query',+ 'db-1'+ );+ expect(dataCollector).toHaveBeenNthCalledWith(+ 2,+ expect.any(Object),+ 'query',+ 'db-2'+ );+ });+ });++ it('does not cache a failed load', async () => {+ await jest.isolateModulesAsync(async () => {+ const { dataCollector } = require('@/lib/platform/common') as { dataCollector: jest.Mock };+ const { consoleLog } = require('@/utils/log') as { consoleLog: jest.Mock };+ jest.clearAllMocks();+ dataCollector+ .mockResolvedValueOnce({ data: [], err: 'DB error' })+ .mockResolvedValueOnce({+ data: [{ id: 'hallucination', default_prompt: 'prompt' }],+ err: null,+ });+ const mod = await import('@/lib/platform/evaluation/evaluation-type-defaults');+ await expect(mod.getEvaluationTypeDefaultPrompts('db-1')).resolves.toEqual({});+ await expect(mod.getEvaluationTypeDefaultPrompts('db-1')).resolves.toEqual({+ hallucination: 'prompt',+ });+ expect(dataCollector).toHaveBeenCalledTimes(2);+ expect(consoleLog).toHaveBeenCalledWith(+ 'Failed to load evaluation type default prompts',+ 'db-1',+ 'DB error'+ );+ });+ });++ it('does not cache a successful read that returns no prompts', async () => {+ await jest.isolateModulesAsync(async () => {+ const { dataCollector } = require('@/lib/platform/common') as { dataCollector: jest.Mock };+ jest.clearAllMocks();+ dataCollector+ .mockResolvedValueOnce({ data: [], err: null })+ .mockResolvedValueOnce({+ data: [{ id: 'bias', default_prompt: 'bias prompt' }],+ err: null,+ });+ const mod = await import('@/lib/platform/evaluation/evaluation-type-defaults');+ await expect(mod.getEvaluationTypeDefaultPrompts('db-1')).resolves.toEqual({});+ await expect(mod.getEvaluationTypeDefaultPrompts('db-1')).resolves.toEqual({+ bias: 'bias prompt',+ });+ expect(dataCollector).toHaveBeenCalledTimes(2);+ });+ });++ it('does not share the no-id cache with an explicit databaseConfigId', async () => {+ await jest.isolateModulesAsync(async () => {+ const { dataCollector } = require('@/lib/platform/common') as { dataCollector: jest.Mock };+ jest.clearAllMocks();+ dataCollector.mockResolvedValue({+ data: [{ id: 'hallucination', default_prompt: 'prompt' }],+ err: null,+ });+ const mod = await import('@/lib/platform/evaluation/evaluation-type-defaults');+ await mod.getEvaluationTypeDefaultPrompts();+ await mod.getEvaluationTypeDefaultPrompts('db-1');+ expect(dataCollector).toHaveBeenCalledTimes(2);+ expect(dataCollector).toHaveBeenNthCalledWith(+ 1,+ expect.any(Object),+ 'query',+ undefined+ );+ });+ });});describe('getEvaluationTypeDefaultPrompt', () => {@@ -123,4 +237,24 @@ describe('getEvaluationTypeDefaultPrompt', () => {);expect(result).toBeUndefined();});++ it('forwards databaseConfigId when loading one prompt', async () => {+ await jest.isolateModulesAsync(async () => {+ const { dataCollector } = require('@/lib/platform/common') as { dataCollector: jest.Mock };+ jest.clearAllMocks();+ dataCollector.mockResolvedValue({+ data: [{ id: 'hallucination', default_prompt: 'hallucination prompt' }],+ err: null,+ });+ const mod = await import('@/lib/platform/evaluation/evaluation-type-defaults');+ await expect(+ mod.getEvaluationTypeDefaultPrompt('hallucination', 'db-9')+ ).resolves.toBe('hallucination prompt');+ expect(dataCollector).toHaveBeenCalledWith(+ expect.any(Object),+ 'query',+ 'db-9'+ );+ });+ });});diff --git a/src/client/src/lib/platform/evaluation/config.ts b/src/client/src/lib/platform/evaluation/config.tsindex 432f4dfb..992ec039 100644--- a/src/client/src/lib/platform/evaluation/config.ts+++ b/src/client/src/lib/platform/evaluation/config.ts@@ -51,9 +51,10 @@ export interface EvaluationTypeWithPrompt {}async function buildEvaluationTypesWithPrompts(- meta: Record<string, any>+ meta: Record<string, any>,+ databaseConfigId?: string): Promise<EvaluationTypeWithPrompt[]> {- const defaultPrompts = await getEvaluationTypeDefaultPrompts();+ const defaultPrompts = await getEvaluationTypeDefaultPrompts(databaseConfigId);const userOverrides = (meta.evaluationTypes as Array<{id: string;enabled?: boolean;@@ -152,7 +153,7 @@ export async function getEvaluationConfig(}const meta = jsonParse((updatedConfig as any).meta || "{}") as Record<string, any>;- const evaluationTypes = await buildEvaluationTypesWithPrompts(meta);+ const evaluationTypes = await buildEvaluationTypesWithPrompts(meta, updatedDBConfig?.id);return {...updatedConfig,@@ -336,7 +337,7 @@ export async function getEvaluationConfigById(const updatedSecretData = (secretData as Secret[])?.[0] || {};const meta = jsonParse((updatedConfig as any).meta || "{}") as Record<string, any>;- const evaluationTypes = await buildEvaluationTypesWithPrompts(meta);+ const evaluationTypes = await buildEvaluationTypesWithPrompts(meta, updatedConfig.databaseConfigId);return {...updatedConfig,@@ -371,7 +372,7 @@ export async function getEvaluationConfigByDbConfigId();const meta = jsonParse((updatedConfig as any).meta || "{}") as Record<string, any>;- const evaluationTypes = await buildEvaluationTypesWithPrompts(meta);+ const evaluationTypes = await buildEvaluationTypesWithPrompts(meta, databaseConfigId);return {...updatedConfig,diff --git a/src/client/src/lib/platform/evaluation/evaluation-type-defaults.ts b/src/client/src/lib/platform/evaluation/evaluation-type-defaults.tsindex 209fd7b2..6f5241ed 100644--- a/src/client/src/lib/platform/evaluation/evaluation-type-defaults.ts+++ b/src/client/src/lib/platform/evaluation/evaluation-type-defaults.ts@@ -1,31 +1,58 @@import { dataCollector } from "@/lib/platform/common";+import { consoleLog } from "@/utils/log";import { OPENLIT_EVALUATION_TYPE_DEFAULTS_TABLE_NAME } from "./table-details";-const CACHE: Record<string, string> = {};-let cacheLoaded = false;+// Per database config. "" is only the no-id fallback; UI, offline, and+// auto-eval all pass the ClickHouse config id they already resolved.+// Evaluations stay on dataCollector (OpenPlait). An explicit id lets the+// no-session paths use getDBConfigByIdForBackground instead of the user session.+const CACHE = new Map<string, Record<string, string>>();-async function loadDefaults(): Promise<Record<string, string>> {- if (cacheLoaded && Object.keys(CACHE).length > 0) return CACHE;- const { data, err } = await dataCollector({- query: `SELECT id, any(default_prompt) AS default_prompt FROM ${OPENLIT_EVALUATION_TYPE_DEFAULTS_TABLE_NAME} GROUP BY id`,- });- if (err || !Array.isArray(data)) return CACHE;+async function loadDefaults(+ databaseConfigId?: string+): Promise<Record<string, string>> {+ const key = databaseConfigId ?? "";+ const cached = CACHE.get(key);+ if (cached) return cached;++ const { data, err } = await dataCollector(+ {+ query: `SELECT id, any(default_prompt) AS default_prompt FROM ${OPENLIT_EVALUATION_TYPE_DEFAULTS_TABLE_NAME} GROUP BY id`,+ },+ "query",+ databaseConfigId+ );++ if (err || !Array.isArray(data)) {+ // Leave the key uncached so the next evaluation retries. The judge+ // otherwise reports this as "Invalid response format".+ consoleLog("Failed to load evaluation type default prompts", key, err);+ return {};+ }++ const result: Record<string, string> = {};for (const r of data as Array<{ id: string; default_prompt: string }>) {- if (r?.id) CACHE[r.id] = r.default_prompt ?? "";+ if (r?.id) result[r.id] = r.default_prompt ?? "";+ }++ // A zero-row read is not a loaded cache. Caching it would pin missing+ // [X evaluation context] blocks until process restart.+ if (Object.keys(result).length > 0) {+ CACHE.set(key, result);}- cacheLoaded = true;- return CACHE;+ return result;}export async function getEvaluationTypeDefaultPrompt(- typeId: string+ typeId: string,+ databaseConfigId?: string): Promise<string | undefined> {- const defaults = await loadDefaults();+ const defaults = await loadDefaults(databaseConfigId);return defaults[typeId];}-export async function getEvaluationTypeDefaultPrompts(): Promise<- Record<string, string>-> {- return loadDefaults();+export async function getEvaluationTypeDefaultPrompts(+ databaseConfigId?: string+): Promise<Record<string, string>> {+ return loadDefaults(databaseConfigId);}