Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

fix(eval): offline evaluations fail after restart — pass databaseConfigId to defaults loader (#1592)

Aftab8a289e638819
diff --git a/src/client/src/__tests__/lib/platform/evaluation/evaluation-config.test.ts b/src/client/src/__tests__/lib/platform/evaluation/evaluation-config.test.ts
index 239184eb..05ec9ba0 100644
--- a/src/client/src/__tests__/lib/platform/evaluation/evaluation-config.test.ts
+++ b/src/client/src/__tests__/lib/platform/evaluation/evaluation-config.test.ts
@@ -220,6 +220,24 @@ describe('getEvaluationConfig', () => {
expect(result.evaluationTypes).toBeDefined();
expect(result.evaluationTypes!.length).toBeGreaterThan(0);
});
+
+ it('loads default prompts for the database config it was given', async () => {
+ (asaw as jest.Mock).mockResolvedValueOnce([null, makeMockEvalConfig()]);
+
+ await getEvaluationConfig({ id: 'db-9' } as any);
+
+ expect(getEvaluationTypeDefaultPrompts).toHaveBeenCalledWith('db-9');
+ });
+
+ it('loads default prompts for the session database config', async () => {
+ (asaw as jest.Mock)
+ .mockResolvedValueOnce([null, { id: 'db-9' }])
+ .mockResolvedValueOnce([null, makeMockEvalConfig()]);
+
+ await getEvaluationConfig();
+
+ expect(getEvaluationTypeDefaultPrompts).toHaveBeenCalledWith('db-9');
+ });
});
// ---------------------------------------------------------------------------
@@ -558,6 +576,15 @@ describe('getEvaluationConfigById', () => {
expect(result.evaluationTypes).toBeDefined();
expect(result.evaluationTypes!.length).toBeGreaterThan(0);
});
+
+ it('loads default prompts for the config row databaseConfigId', async () => {
+ const config = { ...makeMockEvalConfig(), databaseConfigId: 'db-9' };
+ (asaw as jest.Mock).mockResolvedValueOnce([null, config]);
+
+ await getEvaluationConfigById('eval-cfg-1');
+
+ expect(getEvaluationTypeDefaultPrompts).toHaveBeenCalledWith('db-9');
+ });
});
// ---------------------------------------------------------------------------
@@ -609,6 +636,15 @@ describe('getEvaluationConfigByDbConfigId', () => {
expect(result.secret).toEqual(secretWithValue);
});
+ it('loads default prompts for the requested databaseConfigId', async () => {
+ (asaw as jest.Mock).mockResolvedValueOnce([null, makeMockEvalConfig()]);
+ (getSecretById as jest.Mock).mockResolvedValue({ data: [mockSecret] });
+
+ await getEvaluationConfigByDbConfigId('db-9');
+
+ expect(getEvaluationTypeDefaultPrompts).toHaveBeenCalledWith('db-9');
+ });
+
it('defaults meta to an empty object when config.meta is undefined', async () => {
const configWithoutMeta = { ...makeMockEvalConfig(), meta: undefined };
(asaw as jest.Mock).mockResolvedValueOnce([null, configWithoutMeta]);
diff --git a/src/client/src/__tests__/lib/platform/evaluation/evaluation-type-defaults.test.ts b/src/client/src/__tests__/lib/platform/evaluation/evaluation-type-defaults.test.ts
index 56170c5e..3ea47307 100644
--- a/src/client/src/__tests__/lib/platform/evaluation/evaluation-type-defaults.test.ts
+++ b/src/client/src/__tests__/lib/platform/evaluation/evaluation-type-defaults.test.ts
@@ -1,7 +1,8 @@
-// Module-level cache (CACHE + cacheLoaded) means we must isolate modules per test.
+// The prompt cache is module-level, so each test needs a fresh module registry.
// jest.isolateModulesAsync gives each test a fresh module registry.
jest.mock('@/lib/platform/common', () => ({ dataCollector: jest.fn() }));
+jest.mock('@/utils/log', () => ({ consoleLog: jest.fn() }));
jest.mock('@/lib/platform/evaluation/table-details', () => ({
OPENLIT_EVALUATION_TYPE_DEFAULTS_TABLE_NAME: 'openlit_evaluation_type_defaults',
}));
@@ -91,6 +92,119 @@ describe('getEvaluationTypeDefaultPrompts', () => {
expect(dataCollector).toHaveBeenCalledTimes(1);
});
});
+
+ it('passes databaseConfigId through to dataCollector', async () => {
+ await jest.isolateModulesAsync(async () => {
+ const { dataCollector } = require('@/lib/platform/common') as { dataCollector: jest.Mock };
+ jest.clearAllMocks();
+ dataCollector.mockResolvedValue({
+ data: [{ id: 'hallucination', default_prompt: 'prompt' }],
+ err: null,
+ });
+ const mod = await import('@/lib/platform/evaluation/evaluation-type-defaults');
+ await mod.getEvaluationTypeDefaultPrompts('db-1');
+ expect(dataCollector).toHaveBeenCalledWith(
+ expect.objectContaining({
+ query: expect.stringContaining('openlit_evaluation_type_defaults'),
+ }),
+ 'query',
+ 'db-1'
+ );
+ });
+ });
+
+ it('keeps a separate cache entry per databaseConfigId', async () => {
+ await jest.isolateModulesAsync(async () => {
+ const { dataCollector } = require('@/lib/platform/common') as { dataCollector: jest.Mock };
+ jest.clearAllMocks();
+ dataCollector.mockResolvedValue({
+ data: [{ id: 'hallucination', default_prompt: 'prompt' }],
+ err: null,
+ });
+ const mod = await import('@/lib/platform/evaluation/evaluation-type-defaults');
+ await mod.getEvaluationTypeDefaultPrompts('db-1');
+ await mod.getEvaluationTypeDefaultPrompts('db-1');
+ await mod.getEvaluationTypeDefaultPrompts('db-2');
+ expect(dataCollector).toHaveBeenCalledTimes(2);
+ expect(dataCollector).toHaveBeenNthCalledWith(
+ 1,
+ expect.any(Object),
+ 'query',
+ 'db-1'
+ );
+ expect(dataCollector).toHaveBeenNthCalledWith(
+ 2,
+ expect.any(Object),
+ 'query',
+ 'db-2'
+ );
+ });
+ });
+
+ it('does not cache a failed load', async () => {
+ await jest.isolateModulesAsync(async () => {
+ const { dataCollector } = require('@/lib/platform/common') as { dataCollector: jest.Mock };
+ const { consoleLog } = require('@/utils/log') as { consoleLog: jest.Mock };
+ jest.clearAllMocks();
+ dataCollector
+ .mockResolvedValueOnce({ data: [], err: 'DB error' })
+ .mockResolvedValueOnce({
+ data: [{ id: 'hallucination', default_prompt: 'prompt' }],
+ err: null,
+ });
+ const mod = await import('@/lib/platform/evaluation/evaluation-type-defaults');
+ await expect(mod.getEvaluationTypeDefaultPrompts('db-1')).resolves.toEqual({});
+ await expect(mod.getEvaluationTypeDefaultPrompts('db-1')).resolves.toEqual({
+ hallucination: 'prompt',
+ });
+ expect(dataCollector).toHaveBeenCalledTimes(2);
+ expect(consoleLog).toHaveBeenCalledWith(
+ 'Failed to load evaluation type default prompts',
+ 'db-1',
+ 'DB error'
+ );
+ });
+ });
+
+ it('does not cache a successful read that returns no prompts', async () => {
+ await jest.isolateModulesAsync(async () => {
+ const { dataCollector } = require('@/lib/platform/common') as { dataCollector: jest.Mock };
+ jest.clearAllMocks();
+ dataCollector
+ .mockResolvedValueOnce({ data: [], err: null })
+ .mockResolvedValueOnce({
+ data: [{ id: 'bias', default_prompt: 'bias prompt' }],
+ err: null,
+ });
+ const mod = await import('@/lib/platform/evaluation/evaluation-type-defaults');
+ await expect(mod.getEvaluationTypeDefaultPrompts('db-1')).resolves.toEqual({});
+ await expect(mod.getEvaluationTypeDefaultPrompts('db-1')).resolves.toEqual({
+ bias: 'bias prompt',
+ });
+ expect(dataCollector).toHaveBeenCalledTimes(2);
+ });
+ });
+
+ it('does not share the no-id cache with an explicit databaseConfigId', async () => {
+ await jest.isolateModulesAsync(async () => {
+ const { dataCollector } = require('@/lib/platform/common') as { dataCollector: jest.Mock };
+ jest.clearAllMocks();
+ dataCollector.mockResolvedValue({
+ data: [{ id: 'hallucination', default_prompt: 'prompt' }],
+ err: null,
+ });
+ const mod = await import('@/lib/platform/evaluation/evaluation-type-defaults');
+ await mod.getEvaluationTypeDefaultPrompts();
+ await mod.getEvaluationTypeDefaultPrompts('db-1');
+ expect(dataCollector).toHaveBeenCalledTimes(2);
+ expect(dataCollector).toHaveBeenNthCalledWith(
+ 1,
+ expect.any(Object),
+ 'query',
+ undefined
+ );
+ });
+ });
});
describe('getEvaluationTypeDefaultPrompt', () => {
@@ -123,4 +237,24 @@ describe('getEvaluationTypeDefaultPrompt', () => {
);
expect(result).toBeUndefined();
});
+
+ it('forwards databaseConfigId when loading one prompt', async () => {
+ await jest.isolateModulesAsync(async () => {
+ const { dataCollector } = require('@/lib/platform/common') as { dataCollector: jest.Mock };
+ jest.clearAllMocks();
+ dataCollector.mockResolvedValue({
+ data: [{ id: 'hallucination', default_prompt: 'hallucination prompt' }],
+ err: null,
+ });
+ const mod = await import('@/lib/platform/evaluation/evaluation-type-defaults');
+ await expect(
+ mod.getEvaluationTypeDefaultPrompt('hallucination', 'db-9')
+ ).resolves.toBe('hallucination prompt');
+ expect(dataCollector).toHaveBeenCalledWith(
+ expect.any(Object),
+ 'query',
+ 'db-9'
+ );
+ });
+ });
});
diff --git a/src/client/src/lib/platform/evaluation/config.ts b/src/client/src/lib/platform/evaluation/config.ts
index 432f4dfb..992ec039 100644
--- a/src/client/src/lib/platform/evaluation/config.ts
+++ b/src/client/src/lib/platform/evaluation/config.ts
@@ -51,9 +51,10 @@ export interface EvaluationTypeWithPrompt {
}
async function buildEvaluationTypesWithPrompts(
- meta: Record<string, any>
+ meta: Record<string, any>,
+ databaseConfigId?: string
): Promise<EvaluationTypeWithPrompt[]> {
- const defaultPrompts = await getEvaluationTypeDefaultPrompts();
+ const defaultPrompts = await getEvaluationTypeDefaultPrompts(databaseConfigId);
const userOverrides = (meta.evaluationTypes as Array<{
id: string;
enabled?: boolean;
@@ -152,7 +153,7 @@ export async function getEvaluationConfig(
}
const meta = jsonParse((updatedConfig as any).meta || "{}") as Record<string, any>;
- const evaluationTypes = await buildEvaluationTypesWithPrompts(meta);
+ const evaluationTypes = await buildEvaluationTypesWithPrompts(meta, updatedDBConfig?.id);
return {
...updatedConfig,
@@ -336,7 +337,7 @@ export async function getEvaluationConfigById(
const updatedSecretData = (secretData as Secret[])?.[0] || {};
const meta = jsonParse((updatedConfig as any).meta || "{}") as Record<string, any>;
- const evaluationTypes = await buildEvaluationTypesWithPrompts(meta);
+ const evaluationTypes = await buildEvaluationTypesWithPrompts(meta, updatedConfig.databaseConfigId);
return {
...updatedConfig,
@@ -371,7 +372,7 @@ export async function getEvaluationConfigByDbConfigId(
);
const meta = jsonParse((updatedConfig as any).meta || "{}") as Record<string, any>;
- const evaluationTypes = await buildEvaluationTypesWithPrompts(meta);
+ const evaluationTypes = await buildEvaluationTypesWithPrompts(meta, databaseConfigId);
return {
...updatedConfig,
diff --git a/src/client/src/lib/platform/evaluation/evaluation-type-defaults.ts b/src/client/src/lib/platform/evaluation/evaluation-type-defaults.ts
index 209fd7b2..6f5241ed 100644
--- a/src/client/src/lib/platform/evaluation/evaluation-type-defaults.ts
+++ b/src/client/src/lib/platform/evaluation/evaluation-type-defaults.ts
@@ -1,31 +1,58 @@
import { dataCollector } from "@/lib/platform/common";
+import { consoleLog } from "@/utils/log";
import { OPENLIT_EVALUATION_TYPE_DEFAULTS_TABLE_NAME } from "./table-details";
-const CACHE: Record<string, string> = {};
-let cacheLoaded = false;
+// Per database config. "" is only the no-id fallback; UI, offline, and
+// auto-eval all pass the ClickHouse config id they already resolved.
+// Evaluations stay on dataCollector (OpenPlait). An explicit id lets the
+// no-session paths use getDBConfigByIdForBackground instead of the user session.
+const CACHE = new Map<string, Record<string, string>>();
-async function loadDefaults(): Promise<Record<string, string>> {
- if (cacheLoaded && Object.keys(CACHE).length > 0) return CACHE;
- const { data, err } = await dataCollector({
- query: `SELECT id, any(default_prompt) AS default_prompt FROM ${OPENLIT_EVALUATION_TYPE_DEFAULTS_TABLE_NAME} GROUP BY id`,
- });
- if (err || !Array.isArray(data)) return CACHE;
+async function loadDefaults(
+ databaseConfigId?: string
+): Promise<Record<string, string>> {
+ const key = databaseConfigId ?? "";
+ const cached = CACHE.get(key);
+ if (cached) return cached;
+
+ const { data, err } = await dataCollector(
+ {
+ query: `SELECT id, any(default_prompt) AS default_prompt FROM ${OPENLIT_EVALUATION_TYPE_DEFAULTS_TABLE_NAME} GROUP BY id`,
+ },
+ "query",
+ databaseConfigId
+ );
+
+ if (err || !Array.isArray(data)) {
+ // Leave the key uncached so the next evaluation retries. The judge
+ // otherwise reports this as "Invalid response format".
+ consoleLog("Failed to load evaluation type default prompts", key, err);
+ return {};
+ }
+
+ const result: Record<string, string> = {};
for (const r of data as Array<{ id: string; default_prompt: string }>) {
- if (r?.id) CACHE[r.id] = r.default_prompt ?? "";
+ if (r?.id) result[r.id] = r.default_prompt ?? "";
+ }
+
+ // A zero-row read is not a loaded cache. Caching it would pin missing
+ // [X evaluation context] blocks until process restart.
+ if (Object.keys(result).length > 0) {
+ CACHE.set(key, result);
}
- cacheLoaded = true;
- return CACHE;
+ return result;
}
export async function getEvaluationTypeDefaultPrompt(
- typeId: string
+ typeId: string,
+ databaseConfigId?: string
): Promise<string | undefined> {
- const defaults = await loadDefaults();
+ const defaults = await loadDefaults(databaseConfigId);
return defaults[typeId];
}
-export async function getEvaluationTypeDefaultPrompts(): Promise<
- Record<string, string>
-> {
- return loadDefaults();
+export async function getEvaluationTypeDefaultPrompts(
+ databaseConfigId?: string
+): Promise<Record<string, string>> {
+ return loadDefaults(databaseConfigId);
}