Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

feat: add sample rate for auto evaluations (#1309) (#1310)

Vanshika Mehtab1d6f802339b
diff --git a/.github/workflows/ui-test.yml b/.github/workflows/ui-test.yml
index 060e94e9..948ed73b 100644
--- a/.github/workflows/ui-test.yml
+++ b/.github/workflows/ui-test.yml
@@ -38,6 +38,9 @@ jobs:
- name: Generate Prisma client
run: npx prisma generate
+ - name: Type check
+ run: npm run typecheck
+
- name: Run Jest tests
run: npm test -- --coverage --ci --coverageReporters=json-summary --coverageReporters=text
diff --git a/src/client/package.json b/src/client/package.json
index 0d7ac246..a56e7ea5 100644
--- a/src/client/package.json
+++ b/src/client/package.json
@@ -7,6 +7,7 @@
"build": "next build",
"start": "next start",
"lint": "next lint",
+ "typecheck": "tsc --noEmit",
"seed": "npx prisma db seed",
"test": "jest",
"test:watch": "jest --watch",
diff --git a/src/client/src/__tests__/helpers/client/filter.test.ts b/src/client/src/__tests__/helpers/client/filter.test.ts
index cffafcfc..296b3acb 100644
--- a/src/client/src/__tests__/helpers/client/filter.test.ts
+++ b/src/client/src/__tests__/helpers/client/filter.test.ts
@@ -1,7 +1,7 @@
import { getFilterParamsForDashboard } from '@/helpers/client/filter';
const baseFilter = {
- timeLimit: { type: '24H', start: new Date('2024-01-01'), end: new Date('2024-01-02') },
+ timeLimit: { type: '24H' as const, start: new Date('2024-01-01'), end: new Date('2024-01-02') },
limit: 25,
offset: 0,
selectedConfig: { models: ['gpt-4'] },
diff --git a/src/client/src/__tests__/helpers/server/trace.test.ts b/src/client/src/__tests__/helpers/server/trace.test.ts
index 7c6e20e8..6e64efba 100644
--- a/src/client/src/__tests__/helpers/server/trace.test.ts
+++ b/src/client/src/__tests__/helpers/server/trace.test.ts
@@ -7,7 +7,7 @@ describe('buildHierarchy', () => {
it('returns a single root span with no children', () => {
const data = [{ SpanId: 'root', ParentSpanId: '', name: 'root-span' }];
- const result = buildHierarchy(data);
+ const result = buildHierarchy(data) as any;
expect(result).not.toBeNull();
expect(result.SpanId).toBe('root');
expect(result.children).toHaveLength(0);
@@ -19,7 +19,7 @@ describe('buildHierarchy', () => {
{ SpanId: 'child1', ParentSpanId: 'root' },
{ SpanId: 'child2', ParentSpanId: 'root' },
];
- const result = buildHierarchy(data);
+ const result = buildHierarchy(data) as any;
expect(result.SpanId).toBe('root');
expect(result.children).toHaveLength(2);
const childIds = result.children.map((c: any) => c.SpanId);
@@ -33,7 +33,7 @@ describe('buildHierarchy', () => {
{ SpanId: 'child', ParentSpanId: 'root' },
{ SpanId: 'grandchild', ParentSpanId: 'child' },
];
- const result = buildHierarchy(data);
+ const result = buildHierarchy(data) as any;
expect(result.SpanId).toBe('root');
expect(result.children[0].SpanId).toBe('child');
expect(result.children[0].children[0].SpanId).toBe('grandchild');
@@ -44,7 +44,7 @@ describe('buildHierarchy', () => {
{ SpanId: 'root', ParentSpanId: '', Duration: 100, ServiceName: 'api' },
{ SpanId: 'child', ParentSpanId: 'root', Duration: 50, ServiceName: 'db' },
];
- const result = buildHierarchy(data);
+ const result = buildHierarchy(data) as any;
expect(result.Duration).toBe(100);
expect(result.children[0].Duration).toBe(50);
});
@@ -54,7 +54,7 @@ describe('buildHierarchy', () => {
{ SpanId: 'root', ParentSpanId: '' },
{ SpanId: 'orphan', ParentSpanId: 'nonexistent-parent' },
];
- const result = buildHierarchy(data);
+ const result = buildHierarchy(data) as any;
expect(result.SpanId).toBe('root');
expect(result.children).toHaveLength(0);
});
@@ -64,7 +64,7 @@ describe('buildHierarchy', () => {
{ SpanId: 'child', ParentSpanId: 'root' },
{ SpanId: 'root', ParentSpanId: '' },
];
- const result = buildHierarchy(data);
+ const result = buildHierarchy(data) as any;
expect(result.SpanId).toBe('root');
});
});
diff --git a/src/client/src/__tests__/helpers/server/widget.test.ts b/src/client/src/__tests__/helpers/server/widget.test.ts
index fd1607ec..b196a7e6 100644
--- a/src/client/src/__tests__/helpers/server/widget.test.ts
+++ b/src/client/src/__tests__/helpers/server/widget.test.ts
@@ -71,7 +71,7 @@ describe('normalizeWidgetToServer', () => {
properties: {},
config: {},
} as any;
- const result = normalizeWidgetToServer(widget);
+ const result = normalizeWidgetToServer(widget) as any;
expect(result.id).toBe('w1');
expect(result.name).toBe('My Widget');
});
@@ -105,7 +105,7 @@ describe('normalizeWidgetToClient', () => {
properties: '{}',
config: '{}',
} as any;
- const result = normalizeWidgetToClient(widget);
+ const result = normalizeWidgetToClient(widget) as any;
expect(result.id).toBe('w2');
expect(result.name).toBe('Test');
});
diff --git a/src/client/src/__tests__/lib/chat/access.test.ts b/src/client/src/__tests__/lib/chat/access.test.ts
index b4782473..cfcc08c8 100644
--- a/src/client/src/__tests__/lib/chat/access.test.ts
+++ b/src/client/src/__tests__/lib/chat/access.test.ts
@@ -35,7 +35,7 @@ describe("Otter route access wrappers", () => {
const wrapped = wrapper(handler);
expect(wrapped).toBe(handler);
- expect(await wrapped({ method: "GET" }, { params: {} })).toBe(response);
+ expect(await (wrapped as any)({ method: "GET" }, { params: {} })).toBe(response);
expect(handler).toHaveBeenCalledWith(
{ method: "GET" },
{ params: {} }
diff --git a/src/client/src/__tests__/lib/db-config.test.ts b/src/client/src/__tests__/lib/db-config.test.ts
index c79c2ad5..f59b085b 100644
--- a/src/client/src/__tests__/lib/db-config.test.ts
+++ b/src/client/src/__tests__/lib/db-config.test.ts
@@ -129,7 +129,7 @@ describe('getDBConfigByUser', () => {
(prisma.databaseConfigUser.findMany as jest.Mock)
.mockResolvedValueOnce([]) // orphaned configs query
.mockResolvedValueOnce([mockUserConfig]); // user configs query
- const result = await getDBConfigByUser();
+ const result = (await getDBConfigByUser()) as any;
expect(result).toHaveLength(1);
expect(result[0].isCurrent).toBe(true);
expect(result[0].permissions.canEdit).toBe(true);
diff --git a/src/client/src/__tests__/lib/platform/agents/materialize.test.ts b/src/client/src/__tests__/lib/platform/agents/materialize.test.ts
index 70f1b05b..0768144a 100644
--- a/src/client/src/__tests__/lib/platform/agents/materialize.test.ts
+++ b/src/client/src/__tests__/lib/platform/agents/materialize.test.ts
@@ -49,7 +49,7 @@ interface RecordedInsert {
function captureInsertedRows(): RecordedInsert[] {
const inserts: RecordedInsert[] = [];
- mockedDC.mockImplementation(async (config: any, op: string) => {
+ mockedDC.mockImplementation(async (config: any, op?: string) => {
if (op === "insert") {
inserts.push({
table: String(config.table),
@@ -135,7 +135,7 @@ describe("materializeAgents — workload_key dedup", () => {
// `res.err`. The implementation here short-circuits every non-
// insert call with `{ data: [] }` so the conflict query sees
// "no conflicting coding rows" and proceeds to the insert.
- mockedDC.mockImplementation(async (c: any, op: string) => {
+ mockedDC.mockImplementation(async (c: any, op?: string) => {
if (op === "insert") {
inserts.push({
table: String(c.table),
@@ -200,7 +200,7 @@ describe("materializeAgents — workload_key dedup", () => {
// `res.err`. The implementation here short-circuits every non-
// insert call with `{ data: [] }` so the conflict query sees
// "no conflicting coding rows" and proceeds to the insert.
- mockedDC.mockImplementation(async (c: any, op: string) => {
+ mockedDC.mockImplementation(async (c: any, op?: string) => {
if (op === "insert") {
inserts.push({
table: String(c.table),
@@ -234,7 +234,7 @@ describe("materializeAgents — workload_key dedup", () => {
// the workload_key dedup cannot merge them with the controller row
// (their workload_key is empty) and the agent shows up twice.
const queries: string[] = [];
- mockedDC.mockImplementation(async (config: any, op: string) => {
+ mockedDC.mockImplementation(async (config: any, op?: string) => {
if (op === "query") {
queries.push(String(config.query));
return { data: [] } as any;
@@ -296,7 +296,7 @@ describe("materializeAgents — workload_key dedup", () => {
// `res.err`. The implementation here short-circuits every non-
// insert call with `{ data: [] }` so the conflict query sees
// "no conflicting coding rows" and proceeds to the insert.
- mockedDC.mockImplementation(async (c: any, op: string) => {
+ mockedDC.mockImplementation(async (c: any, op?: string) => {
if (op === "insert") {
inserts.push({
table: String(c.table),
@@ -351,7 +351,7 @@ describe("materializeAgents — workload_key dedup", () => {
// `res.err`. The implementation here short-circuits every non-
// insert call with `{ data: [] }` so the conflict query sees
// "no conflicting coding rows" and proceeds to the insert.
- mockedDC.mockImplementation(async (c: any, op: string) => {
+ mockedDC.mockImplementation(async (c: any, op?: string) => {
if (op === "insert") {
inserts.push({
table: String(c.table),
@@ -379,7 +379,7 @@ describe("materializeAgents — workload_key dedup", () => {
// provider logos disappear even when the controller has detected
// imports.
const queries: string[] = [];
- mockedDC.mockImplementation(async (config: any, op: string) => {
+ mockedDC.mockImplementation(async (config: any, op?: string) => {
if (op === "query") {
queries.push(String(config.query));
return { data: [] } as any;
diff --git a/src/client/src/__tests__/lib/platform/agents/version-where.test.ts b/src/client/src/__tests__/lib/platform/agents/version-where.test.ts
index 513aed49..e17602dd 100644
--- a/src/client/src/__tests__/lib/platform/agents/version-where.test.ts
+++ b/src/client/src/__tests__/lib/platform/agents/version-where.test.ts
@@ -18,7 +18,7 @@ describe("buildVersionWhereClause", () => {
const clause = buildVersionWhereClause({
versionHash: "v1",
firstSeen: "2026-01-01T12:34:56.789Z",
- lastSeen: new Date("2026-01-02T01:02:03.000Z"),
+ lastSeen: new Date("2026-01-02T01:02:03.000Z") as any,
hasAttributeSpans: false,
});
diff --git a/src/client/src/__tests__/lib/platform/chat/tools.test.ts b/src/client/src/__tests__/lib/platform/chat/tools.test.ts
index 0ed07efb..aeeb5836 100644
--- a/src/client/src/__tests__/lib/platform/chat/tools.test.ts
+++ b/src/client/src/__tests__/lib/platform/chat/tools.test.ts
@@ -123,7 +123,7 @@ describe("getChatTools", () => {
});
it("builds the expected tool set", () => {
- const tools = getChatTools("user-1", "db-1");
+ const tools = getChatTools("user-1", "db-1") as any;
expect(Object.keys(tools)).toEqual(
expect.arrayContaining([
@@ -150,7 +150,7 @@ describe("getChatTools", () => {
it("creates a rule and adds condition groups when provided", async () => {
(createRule as jest.Mock).mockResolvedValue({ id: "rule-1" });
- const tools = getChatTools("user-1", "db-1");
+ const tools = getChatTools("user-1", "db-1") as any;
await expect(
tools.create_rule.execute({
@@ -174,7 +174,7 @@ describe("getChatTools", () => {
it("returns rule operation errors without throwing", async () => {
(deleteRule as jest.Mock).mockResolvedValue(["delete failed"]);
(getRules as jest.Mock).mockResolvedValue({ err: "list failed" });
- const tools = getChatTools("user-1", "db-1");
+ const tools = getChatTools("user-1", "db-1") as any;
await expect(tools.delete_rule.execute({ id: "rule-1" })).resolves.toEqual({
success: false,
@@ -188,7 +188,7 @@ describe("getChatTools", () => {
it("normalizes vault keys before storing secrets", async () => {
(upsertSecret as jest.Mock).mockResolvedValue(undefined);
- const tools = getChatTools("user-1", "db-1");
+ const tools = getChatTools("user-1", "db-1") as any;
await expect(
tools.create_vault_secret.execute({
@@ -216,7 +216,7 @@ describe("getChatTools", () => {
(getCustomModels as jest.Mock).mockResolvedValue({
data: [{ id: "model-1", model_id: "custom-model", provider: "openai" }],
});
- const tools = getChatTools("user-1", "db-1");
+ const tools = getChatTools("user-1", "db-1") as any;
await expect(
tools.create_custom_model.execute({
@@ -262,7 +262,7 @@ describe("getChatTools", () => {
data: [{ id: "entity-1" }],
err: null,
});
- const tools = getChatTools("user-1", "db-1");
+ const tools = getChatTools("user-1", "db-1") as any;
await expect(
tools.update_rule.execute({ id: "rule-1", name: "Updated" })
@@ -309,7 +309,7 @@ describe("getChatTools", () => {
data: [{ id: "ctx-1", name: "Policy", status: "ACTIVE" }],
err: null,
});
- const tools = getChatTools("user-1", "db-1");
+ const tools = getChatTools("user-1", "db-1") as any;
await expect(
tools.create_context.execute({
@@ -374,7 +374,7 @@ describe("getChatTools", () => {
data: [{ id: "prompt-1", name: "Prompt", totalVersions: 2 }],
err: null,
});
- const tools = getChatTools("user-1", "db-1");
+ const tools = getChatTools("user-1", "db-1") as any;
await expect(
tools.create_prompt.execute({ name: "Prompt", prompt: "Hello {{name}}" })
@@ -433,7 +433,7 @@ describe("getChatTools", () => {
});
it("returns get_prompt validation and lookup errors without mutating prompts", async () => {
- const tools = getChatTools("user-1", "db-1");
+ const tools = getChatTools("user-1", "db-1") as any;
await expect(tools.get_prompt.execute({})).resolves.toEqual({
success: false,
@@ -482,7 +482,7 @@ describe("getChatTools", () => {
data: [{ id: "secret-1", key: "API_KEY", tags: ["llm"] }],
err: null,
});
- const tools = getChatTools("user-1", "db-1");
+ const tools = getChatTools("user-1", "db-1") as any;
await expect(
tools.update_vault_secret.execute({
@@ -518,7 +518,7 @@ describe("getChatTools", () => {
it("executes custom model update and delete tools", async () => {
(updateCustomModel as jest.Mock).mockResolvedValue({ err: null });
(deleteCustomModel as jest.Mock).mockResolvedValue({ err: null });
- const tools = getChatTools("user-1", "db-1");
+ const tools = getChatTools("user-1", "db-1") as any;
await expect(
tools.update_custom_model.execute({
@@ -550,7 +550,7 @@ describe("getChatTools", () => {
(createPrompt as jest.Mock).mockRejectedValue(new Error("prompt failed"));
(upsertSecret as jest.Mock).mockRejectedValue(new Error("secret failed"));
(createCustomModel as jest.Mock).mockRejectedValue(new Error("model failed"));
- const tools = getChatTools("user-1", "db-1");
+ const tools = getChatTools("user-1", "db-1") as any;
await expect(tools.update_rule.execute({ id: "rule-1" })).resolves.toEqual({
success: false,
@@ -607,7 +607,7 @@ describe("getChatTools", () => {
],
},
});
- const tools = getChatTools("user-1", "db-1");
+ const tools = getChatTools("user-1", "db-1") as any;
const result = await tools.analyze_trace.execute({
span_id: "span-root",
@@ -670,7 +670,7 @@ describe("getChatTools", () => {
})}\n`
),
});
- const tools = getChatTools("user-1", "db-1");
+ const tools = getChatTools("user-1", "db-1") as any;
const result = await tools.analyze_traces_by_attribute.execute({
attribute_key: "session.id",
diff --git a/src/client/src/__tests__/lib/platform/evaluation/evaluation-config.test.ts b/src/client/src/__tests__/lib/platform/evaluation/evaluation-config.test.ts
index 667841f4..98e3169b 100644
--- a/src/client/src/__tests__/lib/platform/evaluation/evaluation-config.test.ts
+++ b/src/client/src/__tests__/lib/platform/evaluation/evaluation-config.test.ts
@@ -11,6 +11,7 @@ jest.mock('@/constants/messages', () => ({
DATABASE_CONFIG_NOT_FOUND: 'DB config not found',
EVALUATION_CONFIG_SET_ERROR: 'Eval config set error',
CRON_JOB_UPDATION_ERROR: 'Cron job error',
+ EVALUATION_SAMPLE_RATE_INVALID: 'Sample rate must be a number between 0 and 1.',
})),
}));
jest.mock('@/lib/db-config', () => ({
@@ -73,7 +74,7 @@ import { getSecretById } from '@/lib/platform/vault';
import Cron from '@/helpers/server/cron';
import getMessage from '@/constants/messages';
import { randomUUID } from 'crypto';
-import { jsonParse } from '@/utils/json';
+import { jsonParse, jsonStringify } from '@/utils/json';
const mockDBConfig = { id: 'db-1', name: 'test-db' };
@@ -103,6 +104,7 @@ beforeEach(() => {
DATABASE_CONFIG_NOT_FOUND: 'DB config not found',
EVALUATION_CONFIG_SET_ERROR: 'Eval config set error',
CRON_JOB_UPDATION_ERROR: 'Cron job error',
+ EVALUATION_SAMPLE_RATE_INVALID: 'Sample rate must be a number between 0 and 1.',
});
// Re-apply throwIfError implementation after resetAllMocks
@@ -130,6 +132,7 @@ beforeEach(() => {
(jsonParse as jest.Mock).mockImplementation((v: string) => {
try { return JSON.parse(v); } catch { return {}; }
});
+ (jsonStringify as jest.Mock).mockImplementation((v: unknown) => JSON.stringify(v));
});
// ---------------------------------------------------------------------------
@@ -341,6 +344,71 @@ describe('setEvaluationConfig', () => {
setEvaluationConfig(autoConfig as any, 'http://api.example.com')
).rejects.toThrow('Cron system unavailable');
});
+
+ it('throws when evalSampleRate is invalid', async () => {
+ (asaw as jest.Mock).mockResolvedValueOnce([null, mockDBConfig]);
+
+ await expect(
+ setEvaluationConfig(
+ { auto: false, meta: JSON.stringify({ evalSampleRate: 'invalid' }) } as any,
+ 'http://api.example.com'
+ )
+ ).rejects.toThrow('Sample rate must be a number between 0 and 1.');
+ });
+
+ it('normalizes evalSampleRate when creating config', async () => {
+ const newConfig = {
+ auto: false,
+ recurringTime: '',
+ meta: JSON.stringify({ evalSampleRate: 0.25 }),
+ };
+ const createdRecord = { id: 'new-cfg-1', ...newConfig, databaseConfigId: 'db-1' };
+
+ (asaw as jest.Mock)
+ .mockResolvedValueOnce([null, mockDBConfig])
+ .mockResolvedValueOnce([null, createdRecord]);
+
+ await setEvaluationConfig(newConfig as any, 'http://api.example.com');
+
+ expect(prisma.evaluationConfigs.create).toHaveBeenCalledWith(
+ expect.objectContaining({
+ data: expect.objectContaining({
+ meta: expect.stringContaining('"evalSampleRate":0.25'),
+ }),
+ })
+ );
+ });
+
+ it('merges incoming meta with previous meta on update', async () => {
+ const previousConfig = {
+ ...makeMockEvalConfig(),
+ meta: JSON.stringify({ cronJobId: 'existing-cron', engine: 'old-engine' }),
+ };
+ const inputConfig = {
+ id: 'eval-cfg-1',
+ auto: true,
+ recurringTime: '0 * * * *',
+ meta: JSON.stringify({ engine: 'new-engine', evalSampleRate: 0.5 }),
+ };
+ const updatedRecord = { ...previousConfig, ...inputConfig };
+
+ (asaw as jest.Mock)
+ .mockResolvedValueOnce([null, mockDBConfig])
+ .mockResolvedValueOnce([null, previousConfig])
+ .mockResolvedValueOnce([null, updatedRecord]);
+
+ await setEvaluationConfig(inputConfig as any, 'http://api.example.com');
+
+ expect(prisma.evaluationConfigs.update).toHaveBeenCalledWith(
+ expect.objectContaining({
+ data: expect.objectContaining({
+ meta: expect.stringMatching(
+ /"engine":"new-engine".*"evalSampleRate":0\.5|"evalSampleRate":0\.5.*"engine":"new-engine"/
+ ),
+ }),
+ })
+ );
+ });
});
// ---------------------------------------------------------------------------
diff --git a/src/client/src/__tests__/lib/platform/evaluation/evaluation.test.ts b/src/client/src/__tests__/lib/platform/evaluation/evaluation.test.ts
index d691b07f..90ff99c5 100644
--- a/src/client/src/__tests__/lib/platform/evaluation/evaluation.test.ts
+++ b/src/client/src/__tests__/lib/platform/evaluation/evaluation.test.ts
@@ -84,6 +84,7 @@ import { getDBConfigById } from '@/lib/db-config';
import { getRequestViaSpanId } from '@/lib/platform/request';
import asaw from '@/utils/asaw';
import { runEvaluation } from '@/lib/platform/evaluation/run-evaluation';
+import { CronRunStatus } from '@/types/cron';
beforeEach(() => {
jest.clearAllMocks();
@@ -410,6 +411,145 @@ describe('autoEvaluate', () => {
expect(result.success).toBe(true);
expect(runEvaluation).toHaveBeenCalledTimes(3);
});
+
+ it('does not run evaluation when sample rate is 0', async () => {
+ const evalConfig = {
+ id: 'eval-cfg-1',
+ databaseConfigId: 'db-1',
+ provider: 'openai',
+ model: 'gpt-4',
+ secret: { value: 'sk-1' },
+ meta: JSON.stringify({ evalSampleRate: 0 }),
+ };
+ const trace = { SpanId: 'span-1', Timestamp: '2024-01-01', SpanAttributes: {} };
+
+ (asaw as jest.Mock)
+ .mockResolvedValueOnce([null, evalConfig])
+ .mockResolvedValueOnce([null, { id: 'db-1' }]);
+
+ (dataCollector as jest.Mock)
+ .mockResolvedValueOnce({ data: [trace], err: null })
+ .mockResolvedValue({ data: true, err: null });
+
+ const result = await autoEvaluate(autoEvalConfig as any);
+
+ expect(result.success).toBe(true);
+ expect(runEvaluation).not.toHaveBeenCalled();
+ expect(dataCollector).toHaveBeenCalledWith(
+ expect.objectContaining({
+ table: 'openlit_evaluation',
+ values: [
+ expect.objectContaining({
+ span_id: 'span-1',
+ meta: expect.objectContaining({ source: 'auto_skipped' }),
+ }),
+ ],
+ }),
+ 'insert',
+ 'db-1'
+ );
+ expect(insertCronLog).toHaveBeenCalledWith(
+ expect.objectContaining({
+ runStatus: CronRunStatus.SUCCESS,
+ meta: expect.objectContaining({
+ sampleRate: 0,
+ totalSpans: 1,
+ totalSampled: 0,
+ totalSkipped: 1,
+ totalEvaluated: 0,
+ totalFailed: 0,
+ spanIds: [],
+ }),
+ }),
+ 'db-1'
+ );
+ });
+
+ it('falls back to default sample rate when stored value is invalid', async () => {
+ const evalConfig = {
+ id: 'eval-cfg-1',
+ databaseConfigId: 'db-1',
+ provider: 'openai',
+ model: 'gpt-4',
+ secret: { value: 'sk-1' },
+ meta: JSON.stringify({ evalSampleRate: 'invalid' }),
+ };
+ const trace = { SpanId: 'span-1', Timestamp: '2024-01-01', SpanAttributes: {} };
+
+ (asaw as jest.Mock)
+ .mockResolvedValueOnce([null, evalConfig])
+ .mockResolvedValueOnce([null, { id: 'db-1' }]);
+
+ (dataCollector as jest.Mock)
+ .mockResolvedValueOnce({ data: [trace], err: null })
+ .mockResolvedValue({ data: true, err: null });
+
+ (runEvaluation as jest.Mock).mockResolvedValue({ success: true, result: [] });
+
+ await autoEvaluate(autoEvalConfig as any);
+
+ expect(insertCronLog).toHaveBeenCalledWith(
+ expect.objectContaining({
+ meta: expect.objectContaining({
+ sampleRate: 1,
+ totalSampled: 1,
+ }),
+ }),
+ 'db-1'
+ );
+ });
+
+ it('records sampling metadata in cron log', async () => {
+ const evalConfig = {
+ id: 'eval-cfg-1',
+ databaseConfigId: 'db-1',
+ provider: 'openai',
+ model: 'gpt-4',
+ secret: { value: 'sk-1' },
+ meta: JSON.stringify({ evalSampleRate: 1 }),
+ };
+ const trace = { SpanId: 'span-1', Timestamp: '2024-01-01', SpanAttributes: {} };
+
+ (asaw as jest.Mock)
+ .mockResolvedValueOnce([null, evalConfig])
+ .mockResolvedValueOnce([null, { id: 'db-1' }]);
+
+ (dataCollector as jest.Mock)
+ .mockResolvedValueOnce({ data: [trace], err: null })
+ .mockResolvedValue({ data: true, err: null });
+
+ (runEvaluation as jest.Mock).mockResolvedValue({ success: true, result: [] });
+
+ await autoEvaluate(autoEvalConfig as any);
+
+ expect(insertCronLog).toHaveBeenCalledWith(
+ expect.objectContaining({
+ runStatus: CronRunStatus.SUCCESS,
+ meta: expect.objectContaining({
+ sampleRate: 1,
+ totalSpans: 1,
+ totalSampled: 1,
+ totalSkipped: 0,
+ totalEvaluated: 1,
+ totalFailed: 0,
+ spanIds: ['span-1'],
+ }),
+ }),
+ 'db-1'
+ );
+ });
+
+ it('excludes auto_skipped spans from the pending trace query', async () => {
+ (asaw as jest.Mock)
+ .mockResolvedValueOnce([null, { id: 'eval-cfg-1', databaseConfigId: 'db-1', provider: 'openai', model: 'gpt-4', secret: {} }])
+ .mockResolvedValueOnce([null, { id: 'db-1' }]);
+ (dataCollector as jest.Mock).mockResolvedValue({ data: [], err: null });
+
+ await autoEvaluate(autoEvalConfig as any);
+
+ const [{ query }] = (dataCollector as jest.Mock).mock.calls[0];
+ expect(query).toContain("meta['source'] IN ('auto', 'auto_skipped')");
+ });
});
describe('setEvaluationsForSpanId', () => {
diff --git a/src/client/src/__tests__/lib/platform/evaluation/sampling.test.ts b/src/client/src/__tests__/lib/platform/evaluation/sampling.test.ts
new file mode 100644
index 00000000..b1f71c84
--- /dev/null
+++ b/src/client/src/__tests__/lib/platform/evaluation/sampling.test.ts
@@ -0,0 +1,82 @@
+import {
+ normalizeEvalSampleRate,
+ resolveEvalSampleRate,
+ shouldAutoEvaluateSpan,
+} from "@/lib/platform/evaluation/sampling";
+import {
+ DEFAULT_EVAL_SAMPLE_RATE,
+ evalSampleRateToPercent,
+} from "@/constants/evaluation-sampling";
+
+describe("evalSampleRateToPercent", () => {
+ it("defaults to 100 when value is missing", () => {
+ expect(evalSampleRateToPercent(undefined)).toBe("100");
+ });
+
+ it("preserves fractional percentages without rounding drift", () => {
+ expect(evalSampleRateToPercent(0.333)).toBe("33.3");
+ expect(evalSampleRateToPercent(0.25)).toBe("25");
+ });
+});
+
+describe("normalizeEvalSampleRate", () => {
+ it("defaults to 1 when value is missing", () => {
+ expect(normalizeEvalSampleRate(undefined)).toBe(DEFAULT_EVAL_SAMPLE_RATE);
+ expect(normalizeEvalSampleRate(null)).toBe(DEFAULT_EVAL_SAMPLE_RATE);
+ expect(normalizeEvalSampleRate("")).toBe(DEFAULT_EVAL_SAMPLE_RATE);
+ });
+
+ it("clamps values to the 0-1 range", () => {
+ expect(normalizeEvalSampleRate(0)).toBe(0);
+ expect(normalizeEvalSampleRate(-0.5)).toBe(0);
+ expect(normalizeEvalSampleRate(1)).toBe(1);
+ expect(normalizeEvalSampleRate(2)).toBe(1);
+ expect(normalizeEvalSampleRate(0.25)).toBe(0.25);
+ });
+
+ it("returns NaN for invalid values", () => {
+ expect(Number.isNaN(normalizeEvalSampleRate("invalid"))).toBe(true);
+ expect(Number.isNaN(normalizeEvalSampleRate(Number.NaN))).toBe(true);
+ });
+});
+
+describe("resolveEvalSampleRate", () => {
+ it("falls back to the default when normalization fails", () => {
+ expect(resolveEvalSampleRate("invalid")).toBe(DEFAULT_EVAL_SAMPLE_RATE);
+ expect(resolveEvalSampleRate(undefined)).toBe(DEFAULT_EVAL_SAMPLE_RATE);
+ });
+
+ it("returns normalized values unchanged", () => {
+ expect(resolveEvalSampleRate(0.25)).toBe(0.25);
+ expect(resolveEvalSampleRate(0)).toBe(0);
+ });
+});
+
+describe("shouldAutoEvaluateSpan", () => {
+ it("always evaluates when sample rate is 1", () => {
+ expect(shouldAutoEvaluateSpan("span-a", 1)).toBe(true);
+ expect(shouldAutoEvaluateSpan("span-b", 1.5)).toBe(true);
+ });
+
+ it("never evaluates when sample rate is 0", () => {
+ expect(shouldAutoEvaluateSpan("span-a", 0)).toBe(false);
+ expect(shouldAutoEvaluateSpan("span-b", -0.1)).toBe(false);
+ });
+
+ it("returns a stable decision for the same span id", () => {
+ const first = shouldAutoEvaluateSpan("stable-span-id", 0.5);
+ const second = shouldAutoEvaluateSpan("stable-span-id", 0.5);
+ expect(first).toBe(second);
+ });
+
+ it("samples roughly the requested percentage", () => {
+ const spanIds = Array.from({ length: 1000 }, (_, index) => `span-${index}`);
+ const sampleRate = 0.2;
+ const sampledCount = spanIds.filter((spanId) =>
+ shouldAutoEvaluateSpan(spanId, sampleRate)
+ ).length;
+
+ expect(sampledCount).toBeGreaterThan(120);
+ expect(sampledCount).toBeLessThan(280);
+ });
+});
diff --git a/src/client/src/__tests__/lib/platform/graph-transform.test.ts b/src/client/src/__tests__/lib/platform/graph-transform.test.ts
index 2ccbae56..1b730562 100644
--- a/src/client/src/__tests__/lib/platform/graph-transform.test.ts
+++ b/src/client/src/__tests__/lib/platform/graph-transform.test.ts
@@ -8,7 +8,7 @@ const span = (
SpanId: string,
Timestamp: string,
Duration: number,
- children = []
+ children: any[] = []
) => ({
SpanId,
SpanName: SpanId,
diff --git a/src/client/src/__tests__/lib/platform/rule-engine/rule-engine.test.ts b/src/client/src/__tests__/lib/platform/rule-engine/rule-engine.test.ts
index 6b96590f..ce17dc3d 100644
--- a/src/client/src/__tests__/lib/platform/rule-engine/rule-engine.test.ts
+++ b/src/client/src/__tests__/lib/platform/rule-engine/rule-engine.test.ts
@@ -111,7 +111,7 @@ describe('getRuleById', () => {
it('returns error when ruleResult has err', async () => {
(dataCollector as jest.Mock).mockResolvedValue({ data: null, err: 'DB err' });
- const result = await getRuleById('r1');
+ const result = (await getRuleById('r1')) as any;
expect(result.err).toBe('DB err');
});
diff --git a/src/client/src/__tests__/middleware/check-auth.test.ts b/src/client/src/__tests__/middleware/check-auth.test.ts
index 1dc92099..ab625edf 100644
--- a/src/client/src/__tests__/middleware/check-auth.test.ts
+++ b/src/client/src/__tests__/middleware/check-auth.test.ts
@@ -40,7 +40,7 @@ describe('checkAuth', () => {
beforeEach(() => {
jest.clearAllMocks();
- middleware = checkAuth(nextHandler);
+ middleware = checkAuth(nextHandler as any);
});
it('passes through _next static routes', async () => {
diff --git a/src/client/src/__tests__/selectors/dashboards.test.ts b/src/client/src/__tests__/selectors/dashboards.test.ts
index 143fefaa..390d33fb 100644
--- a/src/client/src/__tests__/selectors/dashboards.test.ts
+++ b/src/client/src/__tests__/selectors/dashboards.test.ts
@@ -15,7 +15,7 @@ const mockStore = {
describe('useDashboardPageSearch', () => {
beforeEach(() => {
mockStore.dashboards.page.search = '';
- (useRootStore as jest.Mock).mockImplementation((selector) => selector(mockStore));
+ (useRootStore as unknown as jest.Mock).mockImplementation((selector) => selector(mockStore));
});
it('returns dashboards.page.search from store', () => {
@@ -30,7 +30,7 @@ describe('useDashboardPageSearch', () => {
describe('useSetDashboardPageSearch', () => {
beforeEach(() => {
- (useRootStore as jest.Mock).mockImplementation((selector) => selector(mockStore));
+ (useRootStore as unknown as jest.Mock).mockImplementation((selector) => selector(mockStore));
});
it('returns dashboards.setPageSearch from store', () => {
diff --git a/src/client/src/__tests__/selectors/filter.test.ts b/src/client/src/__tests__/selectors/filter.test.ts
index dc9ea991..e91ba2b1 100644
--- a/src/client/src/__tests__/selectors/filter.test.ts
+++ b/src/client/src/__tests__/selectors/filter.test.ts
@@ -104,7 +104,7 @@ describe('useFilters', () => {
updateFilter: jest.fn(),
updateConfig: jest.fn(),
};
- (useRootStore as jest.Mock).mockImplementation((selector: (s: any) => any) => {
+ (useRootStore as unknown as jest.Mock).mockImplementation((selector: (s: any) => any) => {
return selector({ filter: mockFilter });
});
const result = useFilters();
diff --git a/src/client/src/__tests__/selectors/page.test.ts b/src/client/src/__tests__/selectors/page.test.ts
index 0333b96d..f5f37c6a 100644
--- a/src/client/src/__tests__/selectors/page.test.ts
+++ b/src/client/src/__tests__/selectors/page.test.ts
@@ -71,7 +71,7 @@ describe('usePageHeader', () => {
it('calls useRootStore twice and returns header and setHeader', () => {
const mockHeader = { title: 'Dashboard', breadcrumbs: [] };
const mockSetHeader = jest.fn();
- (useRootStore as jest.Mock)
+ (useRootStore as unknown as jest.Mock)
.mockReturnValueOnce(mockHeader)
.mockReturnValueOnce(mockSetHeader);
const result = usePageHeader();
diff --git a/src/client/src/__tests__/utils/api.test.ts b/src/client/src/__tests__/utils/api.test.ts
index 5be5a8c1..66de9072 100644
--- a/src/client/src/__tests__/utils/api.test.ts
+++ b/src/client/src/__tests__/utils/api.test.ts
@@ -94,7 +94,7 @@ describe('getData', () => {
it('falls back to first database config when no current config is marked', async () => {
(global.fetch as jest.Mock).mockResolvedValue(makeFetchResponse(true, {}));
- mockSelectedScope([{ id: 'db-1' }, { id: 'db-2' }]);
+ mockSelectedScope([{ id: 'db-1' }, { id: 'db-2' }] as any);
await getData({ url: '/api/test', method: 'GET' });
diff --git a/src/client/src/app/(playground)/evaluations/settings/page.tsx b/src/client/src/app/(playground)/evaluations/settings/page.tsx
index c4ce96b4..f8451cfa 100644
--- a/src/client/src/app/(playground)/evaluations/settings/page.tsx
+++ b/src/client/src/app/(playground)/evaluations/settings/page.tsx
@@ -25,6 +25,7 @@ import Link from "next/link";
import { Zap, Settings2, Clock, Play, Info, SettingsIcon, Layers } from "lucide-react";
import { ProviderMetadata, ModelMetadata } from "@/types/openground";
import { Badge } from "@/components/ui/badge";
+import { evalSampleRateToPercent } from "@/constants/evaluation-sampling";
import FeaturePageHeader from "@/components/(playground)/feature-page-header";
const EvaluationVaultCreate = ({
@@ -54,6 +55,9 @@ export default function EvaluationSettingsPage() {
const [engine] = useState<string>(EVALUATION_ENGINES[0].id);
const [autoEvaluation, setAutoEvaluation] = useState(false);
const [recurringTime, setRecurringTime] = useState("");
+ const [sampleRatePercent, setSampleRatePercent] = useState(
+ evalSampleRateToPercent(undefined)
+ );
const [vaultId, setVaultId] = useState("");
const [vaultKeys, setVaultKeys] = useState<{ label: string; value: string }[]>(
[]
@@ -100,6 +104,8 @@ export default function EvaluationSettingsPage() {
setAutoEvaluation(config.auto || false);
setRecurringTime(config.recurringTime || "");
setVaultId(config.vaultId || "");
+ const meta = JSON.parse(config.meta || "{}");
+ setSampleRatePercent(evalSampleRateToPercent(meta.evalSampleRate));
}
}, [config]);
@@ -119,12 +125,30 @@ export default function EvaluationSettingsPage() {
});
return;
}
+
+ const parsedSampleRate = Number.parseFloat(sampleRatePercent);
+ if (autoEvaluation) {
+ if (
+ !Number.isFinite(parsedSampleRate) ||
+ parsedSampleRate < 0 ||
+ parsedSampleRate > 100
+ ) {
+ toast.error(getMessage().EVALUATION_SAMPLE_RATE_PERCENT_INVALID, {
+ id: EVALUATION_TOAST_ID,
+ });
+ return;
+ }
+ }
+
toast.loading(getMessage().EVALUATION_CONFIG_MODIFYING, {
id: EVALUATION_TOAST_ID,
});
const meta = JSON.parse(config?.meta || "{}");
meta.engine = engine;
+ if (autoEvaluation) {
+ meta.evalSampleRate = parsedSampleRate / 100;
+ }
saveConfig({
body: JSON.stringify({
@@ -374,6 +398,18 @@ export default function EvaluationSettingsPage() {
<p className="text-xs text-stone-500 dark:text-stone-400">
{getMessage().EVALUATION_CRON_HELP}
</p>
+ <Label>{getMessage().EVALUATION_SAMPLE_RATE_LABEL}</Label>
+ <Input
+ type="number"
+ min={0}
+ max={100}
+ step={1}
+ value={sampleRatePercent}
+ onChange={(e) => setSampleRatePercent(e.target.value)}
+ />
+ <p className="text-xs text-stone-500 dark:text-stone-400">
+ {getMessage().EVALUATION_SAMPLE_RATE_DESCRIPTION}
+ </p>
</div>
)}
</CardContent>
diff --git a/src/client/src/constants/evaluation-sampling.ts b/src/client/src/constants/evaluation-sampling.ts
new file mode 100644
index 00000000..d402221c
--- /dev/null
+++ b/src/client/src/constants/evaluation-sampling.ts
@@ -0,0 +1,8 @@
+export const DEFAULT_EVAL_SAMPLE_RATE = 1;
+
+export function evalSampleRateToPercent(rate: number | undefined): string {
+ if (typeof rate !== "number" || !Number.isFinite(rate)) {
+ return String(DEFAULT_EVAL_SAMPLE_RATE * 100);
+ }
+ return String(Number((rate * 100).toFixed(4)));
+}
diff --git a/src/client/src/constants/evaluation-sources.ts b/src/client/src/constants/evaluation-sources.ts
new file mode 100644
index 00000000..a64ab657
--- /dev/null
+++ b/src/client/src/constants/evaluation-sources.ts
@@ -0,0 +1,19 @@
+export const EVALUATION_SOURCE = {
+ AUTO: "auto",
+ AUTO_SKIPPED: "auto_skipped",
+ MANUAL: "manual",
+ MANUAL_FEEDBACK: "manual_feedback",
+} as const;
+
+/** Sources that mark a span as handled by the auto-evaluation cron (evaluated or skipped). */
+export const AUTO_EVALUATION_HANDLED_SOURCES = [
+ EVALUATION_SOURCE.AUTO,
+ EVALUATION_SOURCE.AUTO_SKIPPED,
+] as const;
+
+export function isAutoEvaluationHandledSource(source: unknown): boolean {
+ return (
+ source === EVALUATION_SOURCE.AUTO ||
+ source === EVALUATION_SOURCE.AUTO_SKIPPED
+ );
+}
diff --git a/src/client/src/constants/messages/en.ts b/src/client/src/constants/messages/en.ts
index f73968aa..cd63a93c 100644
--- a/src/client/src/constants/messages/en.ts
+++ b/src/client/src/constants/messages/en.ts
@@ -95,6 +95,13 @@ export const EVALUATION_CRON_SCHEDULE = "Cron schedule";
export const EVALUATION_CRON_PLACEHOLDER = "* * * * *";
export const EVALUATION_CRON_HELP =
"Standard cron expression (e.g. 0 * * * * for hourly)";
+export const EVALUATION_SAMPLE_RATE_LABEL = "Sample rate (%)";
+export const EVALUATION_SAMPLE_RATE_DESCRIPTION =
+ "Percentage of traces to auto-evaluate. Uses deterministic sampling per trace to reduce LLM cost. Traces not selected are marked as skipped and are not re-queried on future cron runs; manual evaluation is always available.";
+export const EVALUATION_SAMPLE_RATE_INVALID =
+ "Sample rate must be a number between 0 and 1.";
+export const EVALUATION_SAMPLE_RATE_PERCENT_INVALID =
+ "Sample rate must be a number between 0 and 100.";
export const EVALUATION_SAVING = "Saving...";
export const EVALUATION_SAVE_CHANGES = "Save Changes";
export const EVALUATION_CREATE_CONFIG = "Create Config";
diff --git a/src/client/src/constants/messages/hi.ts b/src/client/src/constants/messages/hi.ts
index e509dd6d..53ebfb99 100644
--- a/src/client/src/constants/messages/hi.ts
+++ b/src/client/src/constants/messages/hi.ts
@@ -76,6 +76,13 @@ export const EVALUATION_CRON_SCHEDULE = "क्रॉन शेड्यूल";
export const EVALUATION_CRON_PLACEHOLDER = "* * * * *";
export const EVALUATION_CRON_HELP =
"मानक क्रॉन एक्सप्रेशन (उदाहरण: प्रति घंटे के लिए 0 * * * *)";
+export const EVALUATION_SAMPLE_RATE_LABEL = "सैंपल दर (%)";
+export const EVALUATION_SAMPLE_RATE_DESCRIPTION =
+ "ऑटो-इवैल्यूएशन के लिए ट्रेस का प्रतिशत। LLM लागत कम करने के लिए प्रति ट्रेस निर्धारक सैंपलिंग का उपयोग करता है। चयनित नहीं ट्रेस को स्किप के रूप में चिह्नित किया जाता है और भविष्य के क्रॉन रन में दोबारा क्वेरी नहीं होती; मैनुअल इवैल्यूएशन हमेशा उपलब्ध है।";
+export const EVALUATION_SAMPLE_RATE_INVALID =
+ "सैंपल दर 0 और 1 के बीच एक संख्या होनी चाहिए।";
+export const EVALUATION_SAMPLE_RATE_PERCENT_INVALID =
+ "सैंपल दर 0 और 100 के बीच एक संख्या होनी चाहिए।";
export const EVALUATION_SAVING = "सहेज रहे हैं...";
export const EVALUATION_SAVE_CHANGES = "बदलाव सहेजें";
export const EVALUATION_CREATE_CONFIG = "कॉन्फ़िग बनाएं";
diff --git a/src/client/src/lib/platform/evaluation/config.ts b/src/client/src/lib/platform/evaluation/config.ts
index 51126490..ccf99f78 100644
--- a/src/client/src/lib/platform/evaluation/config.ts
+++ b/src/client/src/lib/platform/evaluation/config.ts
@@ -18,6 +18,24 @@ import { randomUUID } from "crypto";
import path, { dirname } from "path";
import { EVALUATION_TYPES } from "@/constants/evaluation-types";
import { getEvaluationTypeDefaultPrompts } from "./evaluation-type-defaults";
+import { normalizeEvalSampleRate } from "./sampling";
+
+function sanitizeEvaluationMeta(meta: Record<string, any>): Record<string, any> {
+ if (!("evalSampleRate" in meta)) {
+ return meta;
+ }
+
+ const normalized = normalizeEvalSampleRate(meta.evalSampleRate);
+ throwIfError(
+ Number.isNaN(normalized),
+ getMessage().EVALUATION_SAMPLE_RATE_INVALID
+ );
+
+ return {
+ ...meta,
+ evalSampleRate: normalized,
+ };
+}
export interface EvaluationTypeWithPrompt {
id: string;
@@ -166,12 +184,22 @@ export async function setEvaluationConfig(
);
evaluationConfigId = previousConfig?.id;
- const meta = jsonParse(previousConfig?.meta || "{}") as Record<string, any>;
- cronJobId = meta?.cronJobId || randomUUID();
- evaluationConfig.meta = jsonStringify({
- ...meta,
- cronJobId,
- });
+ const previousMeta = jsonParse(previousConfig?.meta || "{}") as Record<
+ string,
+ any
+ >;
+ const incomingMeta = jsonParse(evaluationConfig.meta || "{}") as Record<
+ string,
+ any
+ >;
+ cronJobId = previousMeta?.cronJobId || randomUUID();
+ evaluationConfig.meta = jsonStringify(
+ sanitizeEvaluationMeta({
+ ...previousMeta,
+ ...incomingMeta,
+ cronJobId,
+ })
+ );
evaluationConfig = merge(previousConfig, evaluationConfig);
[err, data] = await asaw(
prisma.evaluationConfigs.update({
@@ -184,10 +212,12 @@ export async function setEvaluationConfig(
} else {
cronJobId = randomUUID();
const meta = jsonParse(evaluationConfig.meta) as Record<string, any>;
- evaluationConfig.meta = jsonStringify({
- ...meta,
- cronJobId,
- });
+ evaluationConfig.meta = jsonStringify(
+ sanitizeEvaluationMeta({
+ ...meta,
+ cronJobId,
+ })
+ );
[err, data] = await asaw(
prisma.evaluationConfigs.create({
data: {
diff --git a/src/client/src/lib/platform/evaluation/index.ts b/src/client/src/lib/platform/evaluation/index.ts
index f6535fd8..e6ee3bdb 100644
--- a/src/client/src/lib/platform/evaluation/index.ts
+++ b/src/client/src/lib/platform/evaluation/index.ts
@@ -26,7 +26,16 @@ import { TraceRow } from "@/types/trace";
import { get } from "lodash";
import { getDBConfigById } from "@/lib/db-config";
import { SUPPORTED_EVALUATION_OPERATIONS } from "@/constants/traces";
+import {
+ AUTO_EVALUATION_HANDLED_SOURCES,
+ EVALUATION_SOURCE,
+} from "@/constants/evaluation-sources";
+import { DEFAULT_EVAL_SAMPLE_RATE } from "@/constants/evaluation-sampling";
import { jsonParse } from "@/utils/json";
+import {
+ normalizeEvalSampleRate,
+ shouldAutoEvaluateSpan,
+} from "./sampling";
import {
getLastRunCronLogByCronId,
getLastFailureCronLogBySpanId,
@@ -74,7 +83,7 @@ export async function getEvaluationSummaryForSpanId(spanId: string) {
sum(toFloat64OrZero(meta['cost'])) as totalCost,
argMax(meta['model'], created_at) as latestModel
FROM ${OPENLIT_EVALUATION_TABLE_NAME}
- WHERE span_id = '${sanitizedSpanId}' AND meta['source'] != 'manual_feedback';
+ WHERE span_id = '${sanitizedSpanId}' AND meta['source'] NOT IN ('${EVALUATION_SOURCE.MANUAL_FEEDBACK}', '${EVALUATION_SOURCE.AUTO_SKIPPED}');
`;
const { data, err } = await dataCollector({ query });
@@ -121,10 +130,12 @@ export async function getEvaluationsForSpanId(spanId: string) {
const allRows = (data as any[]) || [];
const feedbackRows = allRows.filter(
- (r: any) => r?.meta?.source === "manual_feedback"
+ (r: any) => r?.meta?.source === EVALUATION_SOURCE.MANUAL_FEEDBACK
);
const aiEvalRows = allRows.filter(
- (r: any) => r?.meta?.source !== "manual_feedback"
+ (r: any) =>
+ r?.meta?.source !== EVALUATION_SOURCE.MANUAL_FEEDBACK &&
+ r?.meta?.source !== EVALUATION_SOURCE.AUTO_SKIPPED
);
const feedbacks: Array<{
createdAt: Date;
@@ -309,6 +320,23 @@ async function storeEvaluation(
return { data: true };
}
+async function storeAutoEvaluationSkip(
+ trace: TraceRow,
+ sampleRate: number,
+ dbConfigId: string
+) {
+ return storeEvaluation(
+ trace.SpanId,
+ [],
+ {
+ source: EVALUATION_SOURCE.AUTO_SKIPPED,
+ traceTimeStamp: String(trace.Timestamp ?? ""),
+ sampleRate: String(sampleRate),
+ },
+ dbConfigId
+ );
+}
+
export async function storeManualFeedback(
spanId: string,
rating: "positive" | "negative" | "neutral",
@@ -570,14 +598,17 @@ export async function autoEvaluate(autoEvaluationConfig: AutoEvaluationConfig) {
const keyPath = `SpanAttributes['${getTraceMappingKeyFullPath("type")}']`;
- // Only exclude traces that already have an auto evaluation (source='auto').
+ // Exclude spans already handled by auto evaluation (evaluated or sampling-skipped).
// Manual feedback does not block auto evaluation. Manual runs are always allowed.
+ const autoHandledSources = AUTO_EVALUATION_HANDLED_SOURCES.map(
+ (source) => `'${source}'`
+ ).join(", ");
const query = `
SELECT t.*
FROM ${OTEL_TRACES_TABLE_NAME} AS t
LEFT JOIN (
SELECT span_id FROM ${OPENLIT_EVALUATION_TABLE_NAME}
- WHERE meta['source'] = 'auto'
+ WHERE meta['source'] IN (${autoHandledSources})
) AS e ON t.SpanId = e.span_id
WHERE ${keyPath} IN (${SUPPORTED_EVALUATION_OPERATIONS.map(
(operation) => `'${operation}'`
@@ -615,10 +646,49 @@ export async function autoEvaluate(autoEvaluationConfig: AutoEvaluationConfig) {
}
const traces = data as TraceRow[];
+ const configMeta = jsonParse(evaluationConfig.meta || "{}") as Record<
+ string,
+ unknown
+ >;
+ const normalizedSampleRate = normalizeEvalSampleRate(
+ configMeta.evalSampleRate
+ );
+ if (Number.isNaN(normalizedSampleRate)) {
+ consoleLog(
+ `Invalid evalSampleRate in evaluation config meta (${String(configMeta.evalSampleRate)}); defaulting to ${DEFAULT_EVAL_SAMPLE_RATE}`
+ );
+ }
+ const sampleRate = Number.isNaN(normalizedSampleRate)
+ ? DEFAULT_EVAL_SAMPLE_RATE
+ : normalizedSampleRate;
+
+ const sampledTraces: TraceRow[] = [];
+ const skippedTraces: TraceRow[] = [];
+ for (const trace of traces) {
+ if (shouldAutoEvaluateSpan(trace.SpanId, sampleRate)) {
+ sampledTraces.push(trace);
+ } else if (sampleRate < 1) {
+ skippedTraces.push(trace);
+ }
+ }
+
+ if (skippedTraces.length > 0) {
+ const skipResults = await Promise.all(
+ skippedTraces.map((trace) =>
+ storeAutoEvaluationSkip(trace, sampleRate, databaseConfig.id)
+ )
+ );
+ for (const skipResult of skipResults) {
+ if (skipResult.err) {
+ consoleLog(skipResult.err);
+ }
+ }
+ }
+
let errorCount = 0;
const results = await Promise.all(
- traces.map(async (trace) => {
+ sampledTraces.map(async (trace) => {
return await getEvaluationConfigForTrace(
trace,
evaluationConfig,
@@ -631,7 +701,7 @@ export async function autoEvaluate(autoEvaluationConfig: AutoEvaluationConfig) {
const errorObject = results.reduce(
(acc: Record<string, string>, r, index) => {
if (!r.success) {
- acc[traces[index].SpanId] = r.error as string;
+ acc[sampledTraces[index].SpanId] = r.error as string;
errorCount++;
}
return acc;
@@ -644,17 +714,20 @@ export async function autoEvaluate(autoEvaluationConfig: AutoEvaluationConfig) {
{
...cronLogObject,
runStatus:
- errorCount === results.length
+ sampledTraces.length > 0 && errorCount === results.length
? CronRunStatus.FAILURE
: errorCount > 0
? CronRunStatus.PARTIAL_SUCCESS
: CronRunStatus.SUCCESS,
errorStacktrace: errorObject,
meta: {
+ sampleRate,
totalSpans: traces.length,
+ totalSampled: sampledTraces.length,
+ totalSkipped: skippedTraces.length,
totalEvaluated: results.length - errorCount,
totalFailed: errorCount,
- spanIds: traces.map((t) => t.SpanId),
+ spanIds: sampledTraces.map((t) => t.SpanId),
},
finishedAt,
duration: differenceInSeconds(finishedAt, startedAt),
diff --git a/src/client/src/lib/platform/evaluation/sampling.ts b/src/client/src/lib/platform/evaluation/sampling.ts
new file mode 100644
index 00000000..c13708c5
--- /dev/null
+++ b/src/client/src/lib/platform/evaluation/sampling.ts
@@ -0,0 +1,51 @@
+import { createHash } from "crypto";
+
+import { DEFAULT_EVAL_SAMPLE_RATE } from "@/constants/evaluation-sampling";
+
+export { DEFAULT_EVAL_SAMPLE_RATE };
+
+export function normalizeEvalSampleRate(value: unknown): number {
+ if (value === undefined || value === null || value === "") {
+ return DEFAULT_EVAL_SAMPLE_RATE;
+ }
+
+ const numeric =
+ typeof value === "number" ? value : Number.parseFloat(String(value));
+
+ if (!Number.isFinite(numeric)) {
+ return Number.NaN;
+ }
+
+ if (numeric <= 0) {
+ return 0;
+ }
+ if (numeric >= 1) {
+ return 1;
+ }
+
+ return numeric;
+}
+
+export function resolveEvalSampleRate(value: unknown): number {
+ const normalized = normalizeEvalSampleRate(value);
+ if (Number.isNaN(normalized)) {
+ return DEFAULT_EVAL_SAMPLE_RATE;
+ }
+ return normalized;
+}
+
+export function shouldAutoEvaluateSpan(
+ spanId: string,
+ sampleRate: number
+): boolean {
+ if (sampleRate >= 1) {
+ return true;
+ }
+ if (sampleRate <= 0) {
+ return false;
+ }
+
+ const hash = createHash("sha256").update(spanId).digest();
+ const normalized = hash.readUInt32BE(0) / 0xffffffff;
+ return normalized < sampleRate;
+}
diff --git a/src/client/types/next-assets.d.ts b/src/client/types/next-assets.d.ts
new file mode 100644
index 00000000..fb8e0cd6
--- /dev/null
+++ b/src/client/types/next-assets.d.ts
@@ -0,0 +1,7 @@
+// Committed type declarations for Next.js static asset imports (e.g. `*.png`,
+// `*.svg`, `*.jpg`). Next normally provides these through the generated
+// `next-env.d.ts`, but that file is gitignored and is not present in CI before
+// a Next build runs, so `tsc --noEmit` would otherwise fail to resolve image
+// imports. Referencing the types here keeps `npm run typecheck` consistent
+// between local and CI environments. Duplicate reference directives are safe.
+/// <reference types="next/image-types/global" />