Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

Feat: use platform-configured evaluation types for offline evaluations (#1382)

Linda Oraegbunam11f177d71fa9
diff --git a/sdk/python/src/openlit/evals/offline.py b/sdk/python/src/openlit/evals/offline.py
index b0422fc4..42aca428 100644
--- a/sdk/python/src/openlit/evals/offline.py
+++ b/sdk/python/src/openlit/evals/offline.py
@@ -350,6 +350,7 @@ def fetch_eval_types(
description=t.get("description", ""),
enabled=t.get("enabled", False),
is_custom=t.get("is_custom", False),
+ threshold_score=t.get("threshold_score"),
)
for t in data.get("eval_types", [])
]
diff --git a/sdk/python/src/openlit/evals/offline_types.py b/sdk/python/src/openlit/evals/offline_types.py
index 936914ee..efa0f582 100644
--- a/sdk/python/src/openlit/evals/offline_types.py
+++ b/sdk/python/src/openlit/evals/offline_types.py
@@ -113,6 +113,7 @@ class EvalType(BaseModel):
description: str = ""
enabled: bool = False
is_custom: bool = False
+ threshold_score: Optional[float] = None
class BatchEvalResult(BaseModel):
diff --git a/sdk/python/tests/test_offline_eval.py b/sdk/python/tests/test_offline_eval.py
index 6dbd3242..2f6a846d 100644
--- a/sdk/python/tests/test_offline_eval.py
+++ b/sdk/python/tests/test_offline_eval.py
@@ -309,6 +309,7 @@ def test_fetch_eval_types(mock_get):
"description": "Detects hallucinated content",
"enabled": True,
"is_custom": False,
+ "threshold_score": 0.7,
},
{
"id": "custom_safety",
@@ -330,8 +331,10 @@ def test_fetch_eval_types(mock_get):
assert len(types) == 2
assert types[0].id == "hallucination"
assert types[0].is_custom is False
+ assert types[0].threshold_score == 0.7
assert types[1].id == "custom_safety"
assert types[1].is_custom is True
+ assert types[1].threshold_score is None
# --- Terminal output ---
diff --git a/sdk/typescript/src/evals/__tests__/offline.test.ts b/sdk/typescript/src/evals/__tests__/offline.test.ts
index 917010eb..c9a58980 100644
--- a/sdk/typescript/src/evals/__tests__/offline.test.ts
+++ b/sdk/typescript/src/evals/__tests__/offline.test.ts
@@ -346,7 +346,14 @@ describe('fetchEvalTypes', () => {
test('returns parsed eval types', async () => {
global.fetch = mockFetchResponse({
eval_types: [
- { id: 'hallucination', label: 'Hallucination', description: 'Detect hallucinated facts', enabled: true, is_custom: false },
+ {
+ id: 'hallucination',
+ label: 'Hallucination',
+ description: 'Detect hallucinated facts',
+ enabled: true,
+ is_custom: false,
+ threshold_score: 0.7,
+ },
{ id: 'custom_1', label: 'Custom', description: 'User-defined', enabled: true, is_custom: true },
],
});
@@ -355,7 +362,9 @@ describe('fetchEvalTypes', () => {
expect(types).toHaveLength(2);
expect(types[0].id).toBe('hallucination');
expect(types[0].isCustom).toBe(false);
+ expect(types[0].thresholdScore).toBe(0.7);
expect(types[1].isCustom).toBe(true);
+ expect(types[1].thresholdScore).toBeUndefined();
});
test('throws on 401', async () => {
diff --git a/sdk/typescript/src/evals/offline.ts b/sdk/typescript/src/evals/offline.ts
index 75873895..5298c1a7 100644
--- a/sdk/typescript/src/evals/offline.ts
+++ b/sdk/typescript/src/evals/offline.ts
@@ -353,6 +353,9 @@ export async function fetchEvalTypes(options: EvalTypesOptions = {}): Promise<Ev
description: t.description || '',
enabled: !!t.enabled,
isCustom: !!t.is_custom,
+ ...(typeof t.threshold_score === 'number'
+ ? { thresholdScore: t.threshold_score }
+ : {}),
}));
} catch (e: any) {
if (e?.name === 'AbortError') {
diff --git a/sdk/typescript/src/evals/types.ts b/sdk/typescript/src/evals/types.ts
index 8a9facf8..3e55947b 100644
--- a/sdk/typescript/src/evals/types.ts
+++ b/sdk/typescript/src/evals/types.ts
@@ -27,6 +27,7 @@ export interface EvalType {
description: string;
enabled: boolean;
isCustom: boolean;
+ thresholdScore?: number;
}
export interface BatchEvalResult {
diff --git a/src/client/src/__tests__/app/api/evaluation/offline/types/[id]/route.test.ts b/src/client/src/__tests__/app/api/evaluation/offline/types/[id]/route.test.ts
new file mode 100644
index 00000000..96e09e14
--- /dev/null
+++ b/src/client/src/__tests__/app/api/evaluation/offline/types/[id]/route.test.ts
@@ -0,0 +1,238 @@
+jest.mock("@/lib/platform/api-keys", () => ({
+ getAPIKeyInfo: jest.fn(),
+}));
+jest.mock("@/lib/platform/evaluation/config", () => ({
+ getEvaluationConfigByDbConfigId: jest.fn(),
+}));
+jest.mock("@/lib/platform/evaluation/sync-rule-entities", () => ({
+ syncRuleEntitiesFromConfig: jest.fn().mockResolvedValue(undefined),
+}));
+jest.mock("@/lib/posthog", () => ({
+ __esModule: true,
+ default: {
+ fireEvent: jest.fn(),
+ },
+}));
+jest.mock("@/constants/messages", () => ({
+ __esModule: true,
+ default: jest.fn(() => ({
+ NO_API_KEY: "No API key provided",
+ EVALUATION_THRESHOLD_SCORE_INVALID:
+ "Threshold score must be a number between 0 and 1.",
+ })),
+}));
+jest.mock("@/utils/json", () => ({
+ jsonParse: jest.fn((v: string) => {
+ try {
+ return JSON.parse(v);
+ } catch {
+ return {};
+ }
+ }),
+ jsonStringify: jest.fn((v: unknown) => JSON.stringify(v)),
+}));
+jest.mock("@/utils/asaw", () =>
+ jest.fn(async (promise: Promise<any>) => {
+ try {
+ const result = await promise;
+ return [null, result];
+ } catch (err) {
+ return [err, null];
+ }
+ })
+);
+jest.mock("@/lib/prisma", () => ({
+ __esModule: true,
+ default: {
+ evaluationConfigs: {
+ update: jest.fn().mockResolvedValue({}),
+ },
+ },
+}));
+
+import { PATCH, OPTIONS } from "@/app/api/evaluation/offline/types/[id]/route";
+import { getAPIKeyInfo } from "@/lib/platform/api-keys";
+import { getEvaluationConfigByDbConfigId } from "@/lib/platform/evaluation/config";
+import prisma from "@/lib/prisma";
+
+class TestHeaders {
+ private values = new Map<string, string>();
+
+ constructor(headers?: Record<string, string>) {
+ Object.entries(headers || {}).forEach(([key, value]) => {
+ this.values.set(key.toLowerCase(), value);
+ });
+ }
+
+ get(key: string) {
+ return this.values.get(key.toLowerCase()) ?? null;
+ }
+}
+
+class TestResponse {
+ status: number;
+ headers: TestHeaders;
+ private body: unknown;
+
+ constructor(
+ body?: unknown,
+ init?: { status?: number; headers?: Record<string, string> }
+ ) {
+ this.body = body;
+ this.status = init?.status ?? 200;
+ this.headers = new TestHeaders(init?.headers);
+ }
+
+ static json(
+ body: unknown,
+ init?: { status?: number; headers?: Record<string, string> }
+ ) {
+ return new TestResponse(body, init);
+ }
+
+ async json() {
+ return this.body;
+ }
+}
+
+(global as any).Response = TestResponse;
+
+function makeRequest({
+ authorization = "Bearer openlit-test-key",
+ body = {},
+}: {
+ authorization?: string;
+ body?: Record<string, unknown>;
+} = {}) {
+ return {
+ headers: {
+ get: (key: string) => {
+ if (key.toLowerCase() === "authorization") return authorization;
+ return null;
+ },
+ },
+ json: jest.fn().mockResolvedValue(body),
+ } as unknown as Request;
+}
+
+function makeInvalidJsonRequest(authorization = "Bearer openlit-test-key") {
+ return {
+ headers: {
+ get: (key: string) => (key.toLowerCase() === "authorization" ? authorization : null),
+ },
+ json: jest.fn().mockRejectedValue(new SyntaxError("Unexpected token")),
+ } as unknown as Request;
+}
+
+describe("PATCH /api/evaluation/offline/types/[id]", () => {
+ beforeEach(() => {
+ jest.clearAllMocks();
+ });
+
+ it("returns 401 when no Bearer token is provided", async () => {
+ const response = await PATCH(makeRequest({ authorization: "" }), {
+ params: { id: "toxicity" },
+ });
+ expect(response.status).toBe(401);
+ });
+
+ it("returns 401 when API key is invalid", async () => {
+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([new Error("Not found"), null]);
+ const response = await PATCH(makeRequest({ body: { enabled: true } }), {
+ params: { id: "toxicity" },
+ });
+ expect(response.status).toBe(401);
+ });
+
+ it("returns 400 on invalid JSON body", async () => {
+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([null, { databaseConfigId: "db-1" }]);
+ const response = await PATCH(makeInvalidJsonRequest(), {
+ params: { id: "toxicity" },
+ });
+ expect(response.status).toBe(400);
+ const data = await response.json();
+ expect(data.err).toBe("Invalid JSON body");
+ });
+
+ it("rejects a non-numeric thresholdScore without calling getEvaluationConfigByDbConfigId", async () => {
+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([null, { databaseConfigId: "db-1" }]);
+ const response = await PATCH(
+ makeRequest({ body: { thresholdScore: "bad" } }),
+ { params: { id: "toxicity" } }
+ );
+ expect(response.status).toBe(400);
+ const data = await response.json();
+ expect(data.err).toBe("Threshold score must be a number between 0 and 1.");
+ expect(getEvaluationConfigByDbConfigId).not.toHaveBeenCalled();
+ });
+
+ it("returns 400 when evaluation config is not found", async () => {
+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([null, { databaseConfigId: "db-1" }]);
+ (getEvaluationConfigByDbConfigId as jest.Mock).mockRejectedValue(new Error("not found"));
+
+ const response = await PATCH(makeRequest({ body: { enabled: true } }), {
+ params: { id: "toxicity" },
+ });
+ expect(response.status).toBe(400);
+ });
+
+ it("scopes the update to the API key's own database config and clamps thresholdScore", async () => {
+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([null, { databaseConfigId: "db-1" }]);
+ (getEvaluationConfigByDbConfigId as jest.Mock).mockResolvedValue({
+ id: "cfg-1",
+ meta: JSON.stringify({
+ evaluationTypes: [{ id: "toxicity", enabled: true, rules: [] }],
+ }),
+ });
+
+ const response = await PATCH(
+ makeRequest({ body: { thresholdScore: 1.5 } }),
+ { params: { id: "toxicity" } }
+ );
+
+ expect(response.status).toBe(200);
+ const data = await response.json();
+ expect(data.data.thresholdScore).toBe(1);
+ expect(getEvaluationConfigByDbConfigId).toHaveBeenCalledWith("db-1", true);
+
+ const updateCall = (prisma.evaluationConfigs.update as jest.Mock).mock.calls[0][0];
+ expect(updateCall.where).toEqual({ id: "cfg-1" });
+ const persistedMeta = JSON.parse(updateCall.data.meta);
+ expect(persistedMeta.evaluationTypes[0]).toMatchObject({
+ id: "toxicity",
+ thresholdScore: 1,
+ });
+ });
+
+ it("creates a new type entry when the id doesn't already exist", async () => {
+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([null, { databaseConfigId: "db-1" }]);
+ (getEvaluationConfigByDbConfigId as jest.Mock).mockResolvedValue({
+ id: "cfg-1",
+ meta: JSON.stringify({ evaluationTypes: [] }),
+ });
+
+ const response = await PATCH(
+ makeRequest({
+ body: { isCustom: true, label: "Custom Check", enabled: true },
+ }),
+ { params: { id: "custom_check" } }
+ );
+
+ expect(response.status).toBe(200);
+ const data = await response.json();
+ expect(data.data).toMatchObject({
+ id: "custom_check",
+ isCustom: true,
+ label: "Custom Check",
+ enabled: true,
+ });
+ });
+});
+
+describe("OPTIONS /api/evaluation/offline/types/[id]", () => {
+ it("allows PATCH and OPTIONS", async () => {
+ const response = await OPTIONS();
+ expect(response.status).toBe(200);
+ expect(response.headers.get("Access-Control-Allow-Methods")).toContain("PATCH");
+ });
+});
diff --git a/src/client/src/__tests__/app/api/evaluation/offline/types/route.test.ts b/src/client/src/__tests__/app/api/evaluation/offline/types/route.test.ts
new file mode 100644
index 00000000..46fd3e84
--- /dev/null
+++ b/src/client/src/__tests__/app/api/evaluation/offline/types/route.test.ts
@@ -0,0 +1,303 @@
+jest.mock("@/lib/platform/api-keys", () => ({
+ getAPIKeyInfo: jest.fn(),
+}));
+jest.mock("@/lib/platform/evaluation/config", () => ({
+ getEvaluationConfigByDbConfigId: jest.fn(),
+}));
+jest.mock("@/lib/platform/evaluation/sync-rule-entities", () => ({
+ syncRuleEntitiesFromConfig: jest.fn().mockResolvedValue(undefined),
+}));
+jest.mock("@/lib/posthog", () => ({
+ __esModule: true,
+ default: {
+ fireEvent: jest.fn(),
+ },
+}));
+jest.mock("@/constants/messages", () => ({
+ __esModule: true,
+ default: jest.fn(() => ({
+ NO_API_KEY: "No API key provided",
+ EVALUATION_THRESHOLD_SCORE_INVALID:
+ "Threshold score must be a number between 0 and 1.",
+ })),
+}));
+jest.mock("@/utils/json", () => ({
+ jsonParse: jest.fn((v: string) => {
+ try {
+ return JSON.parse(v);
+ } catch {
+ return {};
+ }
+ }),
+ jsonStringify: jest.fn((v: unknown) => JSON.stringify(v)),
+}));
+jest.mock("@/utils/asaw", () =>
+ jest.fn(async (promise: Promise<any>) => {
+ try {
+ const result = await promise;
+ return [null, result];
+ } catch (err) {
+ return [err, null];
+ }
+ })
+);
+jest.mock("@/lib/prisma", () => ({
+ __esModule: true,
+ default: {
+ evaluationConfigs: {
+ update: jest.fn().mockResolvedValue({}),
+ },
+ },
+}));
+
+import { GET, POST, OPTIONS } from "@/app/api/evaluation/offline/types/route";
+import { getAPIKeyInfo } from "@/lib/platform/api-keys";
+import { getEvaluationConfigByDbConfigId } from "@/lib/platform/evaluation/config";
+import prisma from "@/lib/prisma";
+
+class TestHeaders {
+ private values = new Map<string, string>();
+
+ constructor(headers?: Record<string, string>) {
+ Object.entries(headers || {}).forEach(([key, value]) => {
+ this.values.set(key.toLowerCase(), value);
+ });
+ }
+
+ get(key: string) {
+ return this.values.get(key.toLowerCase()) ?? null;
+ }
+}
+
+class TestResponse {
+ status: number;
+ headers: TestHeaders;
+ private body: unknown;
+
+ constructor(
+ body?: unknown,
+ init?: { status?: number; headers?: Record<string, string> }
+ ) {
+ this.body = body;
+ this.status = init?.status ?? 200;
+ this.headers = new TestHeaders(init?.headers);
+ }
+
+ static json(
+ body: unknown,
+ init?: { status?: number; headers?: Record<string, string> }
+ ) {
+ return new TestResponse(body, init);
+ }
+
+ async json() {
+ return this.body;
+ }
+}
+
+(global as any).Response = TestResponse;
+
+function makeRequest({
+ authorization = "Bearer openlit-test-key",
+ body = {},
+}: {
+ authorization?: string;
+ body?: Record<string, unknown>;
+} = {}) {
+ return {
+ headers: {
+ get: (key: string) => {
+ if (key.toLowerCase() === "authorization") return authorization;
+ return null;
+ },
+ },
+ json: jest.fn().mockResolvedValue(body),
+ } as unknown as Request;
+}
+
+function makeInvalidJsonRequest(authorization = "Bearer openlit-test-key") {
+ return {
+ headers: {
+ get: (key: string) => (key.toLowerCase() === "authorization" ? authorization : null),
+ },
+ json: jest.fn().mockRejectedValue(new SyntaxError("Unexpected token")),
+ } as unknown as Request;
+}
+
+describe("GET /api/evaluation/offline/types", () => {
+ beforeEach(() => {
+ jest.clearAllMocks();
+ });
+
+ it("returns 401 when no Bearer token is provided", async () => {
+ const response = await GET(makeRequest({ authorization: "" }));
+ expect(response.status).toBe(401);
+ });
+
+ it("returns 401 when API key is invalid", async () => {
+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([new Error("Not found"), null]);
+ const response = await GET(makeRequest());
+ expect(response.status).toBe(401);
+ });
+
+ it("returns configured:false when evaluation config is not found", async () => {
+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([null, { databaseConfigId: "db-1" }]);
+ (getEvaluationConfigByDbConfigId as jest.Mock).mockRejectedValue(new Error("not found"));
+
+ const response = await GET(makeRequest());
+ expect(response.status).toBe(200);
+ const data = await response.json();
+ expect(data.configured).toBe(false);
+ expect(data.eval_types).toEqual([]);
+ });
+
+ it("returns the configured evaluation types", async () => {
+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([null, { databaseConfigId: "db-1" }]);
+ (getEvaluationConfigByDbConfigId as jest.Mock).mockResolvedValue({
+ id: "cfg-1",
+ evaluationTypes: [
+ {
+ id: "hallucination",
+ label: "Hallucination",
+ enabled: true,
+ isCustom: false,
+ thresholdScore: 0.7,
+ },
+ {
+ id: "toxicity",
+ label: "Toxicity",
+ enabled: true,
+ isCustom: false,
+ },
+ ],
+ });
+
+ const response = await GET(makeRequest());
+ expect(response.status).toBe(200);
+ const data = await response.json();
+ expect(data.eval_types).toEqual([
+ {
+ id: "hallucination",
+ label: "Hallucination",
+ description: "",
+ enabled: true,
+ is_custom: false,
+ threshold_score: 0.7,
+ },
+ {
+ id: "toxicity",
+ label: "Toxicity",
+ description: "",
+ enabled: true,
+ is_custom: false,
+ },
+ ]);
+ });
+});
+
+describe("POST /api/evaluation/offline/types", () => {
+ beforeEach(() => {
+ jest.clearAllMocks();
+ });
+
+ it("returns 401 when no Bearer token is provided", async () => {
+ const response = await POST(makeRequest({ authorization: "" }));
+ expect(response.status).toBe(401);
+ });
+
+ it("returns 401 when API key is invalid", async () => {
+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([new Error("Not found"), null]);
+ const response = await POST(makeRequest({ body: { types: [] } }));
+ expect(response.status).toBe(401);
+ });
+
+ it("returns 400 on invalid JSON body", async () => {
+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([null, { databaseConfigId: "db-1" }]);
+ const response = await POST(makeInvalidJsonRequest());
+ expect(response.status).toBe(400);
+ const data = await response.json();
+ expect(data.err).toBe("Invalid JSON body");
+ });
+
+ it("returns 400 when types is not an array", async () => {
+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([null, { databaseConfigId: "db-1" }]);
+ const response = await POST(makeRequest({ body: { types: "not-an-array" } }));
+ expect(response.status).toBe(400);
+ const data = await response.json();
+ expect(data.err).toBe("Invalid types array");
+ });
+
+ it("rejects a non-numeric thresholdScore without calling getEvaluationConfigByDbConfigId", async () => {
+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([null, { databaseConfigId: "db-1" }]);
+ const response = await POST(
+ makeRequest({
+ body: { types: [{ id: "toxicity", enabled: true, thresholdScore: "bad" }] },
+ })
+ );
+ expect(response.status).toBe(400);
+ const data = await response.json();
+ expect(data.err).toBe("Threshold score must be a number between 0 and 1.");
+ expect(getEvaluationConfigByDbConfigId).not.toHaveBeenCalled();
+ });
+
+ it("returns 400 when evaluation config is not found", async () => {
+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([null, { databaseConfigId: "db-1" }]);
+ (getEvaluationConfigByDbConfigId as jest.Mock).mockRejectedValue(new Error("not found"));
+
+ const response = await POST(
+ makeRequest({ body: { types: [{ id: "hallucination", enabled: true }] } })
+ );
+ expect(response.status).toBe(400);
+ const data = await response.json();
+ expect(data.err).toBe(
+ "Evaluation not configured. Set up evaluation in the OpenLIT dashboard first."
+ );
+ });
+
+ it("normalizes and upserts types scoped to the API key's own database config", async () => {
+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([null, { databaseConfigId: "db-1" }]);
+ (getEvaluationConfigByDbConfigId as jest.Mock).mockResolvedValue({
+ id: "cfg-1",
+ meta: JSON.stringify({
+ evaluationTypes: [
+ { id: "bias", enabled: true, rules: [], thresholdScore: 0.4 },
+ ],
+ }),
+ });
+
+ const response = await POST(
+ makeRequest({
+ body: {
+ types: [{ id: "toxicity", enabled: true, thresholdScore: 1.5 }],
+ },
+ })
+ );
+
+ expect(response.status).toBe(200);
+ const data = await response.json();
+ expect(data.data).toEqual([
+ expect.objectContaining({ id: "toxicity", enabled: true, thresholdScore: 1 }),
+ ]);
+ expect(getEvaluationConfigByDbConfigId).toHaveBeenCalledWith("db-1", true);
+ const updateCall = (prisma.evaluationConfigs.update as jest.Mock).mock.calls[0][0];
+ expect(updateCall.where).toEqual({ id: "cfg-1" });
+ const persistedMeta = JSON.parse(updateCall.data.meta);
+ // Partial POST upserts without wiping unrelated types/thresholds.
+ expect(persistedMeta.evaluationTypes).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ id: "bias", thresholdScore: 0.4 }),
+ expect.objectContaining({ id: "toxicity", thresholdScore: 1 }),
+ ])
+ );
+ expect(persistedMeta.evaluationTypes).toHaveLength(2);
+ });
+});
+
+describe("OPTIONS /api/evaluation/offline/types", () => {
+ it("allows GET, POST, and OPTIONS", async () => {
+ const response = await OPTIONS();
+ expect(response.status).toBe(200);
+ expect(response.headers.get("Access-Control-Allow-Methods")).toContain("POST");
+ expect(response.headers.get("Access-Control-Allow-Methods")).toContain("GET");
+ });
+});
diff --git a/src/client/src/__tests__/app/api/evaluation/types/[id]/route.test.ts b/src/client/src/__tests__/app/api/evaluation/types/[id]/route.test.ts
new file mode 100644
index 00000000..edc0f163
--- /dev/null
+++ b/src/client/src/__tests__/app/api/evaluation/types/[id]/route.test.ts
@@ -0,0 +1,184 @@
+jest.mock("@/lib/platform/evaluation/config", () => ({
+ getEvaluationConfig: jest.fn(),
+}));
+jest.mock("@/lib/platform/evaluation/sync-rule-entities", () => ({
+ syncRuleEntitiesFromConfig: jest.fn().mockResolvedValue(undefined),
+}));
+jest.mock("@/lib/posthog", () => ({
+ __esModule: true,
+ default: {
+ fireEvent: jest.fn(),
+ },
+}));
+jest.mock("@/constants/messages", () => ({
+ __esModule: true,
+ default: jest.fn(() => ({
+ EVALUATION_THRESHOLD_SCORE_INVALID:
+ "Threshold score must be a number between 0 and 1.",
+ })),
+}));
+jest.mock("@/utils/json", () => ({
+ jsonParse: jest.fn((v: string) => {
+ try {
+ return JSON.parse(v);
+ } catch {
+ return {};
+ }
+ }),
+ jsonStringify: jest.fn((v: unknown) => JSON.stringify(v)),
+}));
+jest.mock("@/utils/asaw", () =>
+ jest.fn(async (promise: Promise<any>) => {
+ try {
+ const result = await promise;
+ return [null, result];
+ } catch (err) {
+ return [err, null];
+ }
+ })
+);
+jest.mock("@/lib/prisma", () => ({
+ __esModule: true,
+ default: {
+ evaluationConfigs: {
+ update: jest.fn().mockResolvedValue({}),
+ },
+ },
+}));
+
+import { PATCH } from "@/app/api/evaluation/types/[id]/route";
+import { getEvaluationConfig } from "@/lib/platform/evaluation/config";
+import prisma from "@/lib/prisma";
+
+class TestResponse {
+ status: number;
+ private body: unknown;
+
+ constructor(body?: unknown, init?: { status?: number }) {
+ this.body = body;
+ this.status = init?.status ?? 200;
+ }
+
+ static json(body: unknown, init?: { status?: number }) {
+ return new TestResponse(body, init);
+ }
+
+ async json() {
+ return this.body;
+ }
+}
+
+(global as any).Response = TestResponse;
+
+function makeRequest(body: Record<string, unknown>) {
+ return {
+ json: jest.fn().mockResolvedValue(body),
+ } as unknown as Request;
+}
+
+function makeInvalidJsonRequest() {
+ return {
+ json: jest.fn().mockRejectedValue(new SyntaxError("Unexpected token")),
+ } as unknown as Request;
+}
+
+describe("PATCH /api/evaluation/types/[id]", () => {
+ beforeEach(() => {
+ jest.clearAllMocks();
+ });
+
+ it("returns 400 on invalid JSON body", async () => {
+ const response = await PATCH(makeInvalidJsonRequest() as any, {
+ params: { id: "toxicity" },
+ });
+ expect(response.status).toBe(400);
+ const data = await response.json();
+ expect(data.err).toBe("Invalid JSON body");
+ });
+
+ it("rejects a non-numeric thresholdScore without calling getEvaluationConfig", async () => {
+ const response = await PATCH(
+ makeRequest({ thresholdScore: "not-a-number" }) as any,
+ { params: { id: "toxicity" } }
+ );
+
+ expect(response.status).toBe(400);
+ const data = await response.json();
+ expect(data.err).toBe("Threshold score must be a number between 0 and 1.");
+ expect(getEvaluationConfig).not.toHaveBeenCalled();
+ });
+
+ it("returns 400 when evaluation config is not found", async () => {
+ (getEvaluationConfig as jest.Mock).mockRejectedValue(new Error("not found"));
+
+ const response = await PATCH(makeRequest({ enabled: true }) as any, {
+ params: { id: "toxicity" },
+ });
+
+ expect(response.status).toBe(400);
+ const data = await response.json();
+ expect(data.err).toBe("Evaluation config not found");
+ });
+
+ it("clamps an out-of-range thresholdScore and persists it for an existing type", async () => {
+ (getEvaluationConfig as jest.Mock).mockResolvedValue({
+ id: "cfg-1",
+ meta: JSON.stringify({
+ evaluationTypes: [{ id: "toxicity", enabled: true, rules: [] }],
+ }),
+ });
+
+ const response = await PATCH(
+ makeRequest({ thresholdScore: 1.5 }) as any,
+ { params: { id: "toxicity" } }
+ );
+
+ expect(response.status).toBe(200);
+ const data = await response.json();
+ expect(data.data.thresholdScore).toBe(1);
+
+ const updateCall = (prisma.evaluationConfigs.update as jest.Mock).mock.calls[0][0];
+ const persistedMeta = JSON.parse(updateCall.data.meta);
+ expect(persistedMeta.evaluationTypes[0]).toMatchObject({
+ id: "toxicity",
+ thresholdScore: 1,
+ });
+ });
+
+ it("preserves the existing thresholdScore when the body omits it", async () => {
+ (getEvaluationConfig as jest.Mock).mockResolvedValue({
+ id: "cfg-1",
+ meta: JSON.stringify({
+ evaluationTypes: [
+ { id: "toxicity", enabled: true, rules: [], thresholdScore: 0.8 },
+ ],
+ }),
+ });
+
+ const response = await PATCH(makeRequest({ enabled: false }) as any, {
+ params: { id: "toxicity" },
+ });
+
+ const data = await response.json();
+ expect(data.data.thresholdScore).toBe(0.8);
+ });
+
+ it("clears the existing thresholdScore when the body sends null", async () => {
+ (getEvaluationConfig as jest.Mock).mockResolvedValue({
+ id: "cfg-1",
+ meta: JSON.stringify({
+ evaluationTypes: [
+ { id: "toxicity", enabled: true, rules: [], thresholdScore: 0.8 },
+ ],
+ }),
+ });
+
+ const response = await PATCH(
+ makeRequest({ thresholdScore: null }) as any,
+ { params: { id: "toxicity" } }
+ );
+
+ const data = await response.json();
+ expect(data.data.thresholdScore).toBeUndefined();
+ });
+});
diff --git a/src/client/src/__tests__/app/api/evaluation/types/route.test.ts b/src/client/src/__tests__/app/api/evaluation/types/route.test.ts
new file mode 100644
index 00000000..993d6e72
--- /dev/null
+++ b/src/client/src/__tests__/app/api/evaluation/types/route.test.ts
@@ -0,0 +1,202 @@
+jest.mock("@/lib/platform/evaluation/config", () => ({
+ getEvaluationConfig: jest.fn(),
+}));
+jest.mock("@/lib/platform/evaluation/sync-rule-entities", () => ({
+ syncRuleEntitiesFromConfig: jest.fn().mockResolvedValue(undefined),
+}));
+jest.mock("@/lib/posthog", () => ({
+ __esModule: true,
+ default: {
+ fireEvent: jest.fn(),
+ },
+}));
+jest.mock("@/constants/messages", () => ({
+ __esModule: true,
+ default: jest.fn(() => ({
+ EVALUATION_THRESHOLD_SCORE_INVALID:
+ "Threshold score must be a number between 0 and 1.",
+ })),
+}));
+jest.mock("@/utils/json", () => ({
+ jsonParse: jest.fn((v: string) => {
+ try {
+ return JSON.parse(v);
+ } catch {
+ return {};
+ }
+ }),
+ jsonStringify: jest.fn((v: unknown) => JSON.stringify(v)),
+}));
+jest.mock("@/utils/asaw", () =>
+ jest.fn(async (promise: Promise<any>) => {
+ try {
+ const result = await promise;
+ return [null, result];
+ } catch (err) {
+ return [err, null];
+ }
+ })
+);
+jest.mock("@/lib/prisma", () => ({
+ __esModule: true,
+ default: {
+ evaluationConfigs: {
+ update: jest.fn().mockResolvedValue({}),
+ },
+ },
+}));
+
+import { GET, POST } from "@/app/api/evaluation/types/route";
+import { getEvaluationConfig } from "@/lib/platform/evaluation/config";
+import { syncRuleEntitiesFromConfig } from "@/lib/platform/evaluation/sync-rule-entities";
+import prisma from "@/lib/prisma";
+
+class TestResponse {
+ status: number;
+ private body: unknown;
+
+ constructor(body?: unknown, init?: { status?: number }) {
+ this.body = body;
+ this.status = init?.status ?? 200;
+ }
+
+ static json(body: unknown, init?: { status?: number }) {
+ return new TestResponse(body, init);
+ }
+
+ async json() {
+ return this.body;
+ }
+}
+
+(global as any).Response = TestResponse;
+
+function makeRequest(body: Record<string, unknown>) {
+ return {
+ json: jest.fn().mockResolvedValue(body),
+ } as unknown as Request;
+}
+
+function makeInvalidJsonRequest() {
+ return {
+ json: jest.fn().mockRejectedValue(new SyntaxError("Unexpected token")),
+ } as unknown as Request;
+}
+
+describe("GET /api/evaluation/types", () => {
+ beforeEach(() => {
+ jest.clearAllMocks();
+ });
+
+ it("returns the configured evaluation types", async () => {
+ (getEvaluationConfig as jest.Mock).mockResolvedValue({
+ id: "cfg-1",
+ evaluationTypes: [{ id: "hallucination", enabled: true }],
+ });
+
+ const response = await GET(undefined as any);
+ expect(response.status).toBe(200);
+ const data = await response.json();
+ expect(data.data).toEqual([{ id: "hallucination", enabled: true }]);
+ });
+
+ it("returns empty data with an error when config is not found", async () => {
+ (getEvaluationConfig as jest.Mock).mockRejectedValue(new Error("not found"));
+
+ const response = await GET(undefined as any);
+ expect(response.status).toBe(200);
+ const data = await response.json();
+ expect(data.data).toEqual([]);
+ expect(data.err).toBeDefined();
+ });
+});
+
+describe("POST /api/evaluation/types", () => {
+ beforeEach(() => {
+ jest.clearAllMocks();
+ });
+
+ it("returns 400 on invalid JSON body", async () => {
+ const response = await POST(makeInvalidJsonRequest() as any);
+ expect(response.status).toBe(400);
+ const data = await response.json();
+ expect(data.err).toBe("Invalid JSON body");
+ });
+
+ it("returns 400 when types is not an array", async () => {
+ const response = await POST(makeRequest({ types: "not-an-array" }) as any);
+ expect(response.status).toBe(400);
+ const data = await response.json();
+ expect(data.err).toBe("Invalid types array");
+ });
+
+ it("rejects a non-numeric thresholdScore without calling getEvaluationConfig", async () => {
+ const response = await POST(
+ makeRequest({
+ types: [{ id: "toxicity", enabled: true, thresholdScore: "not-a-number" }],
+ }) as any
+ );
+
+ expect(response.status).toBe(400);
+ const data = await response.json();
+ expect(data.err).toBe("Threshold score must be a number between 0 and 1.");
+ expect(getEvaluationConfig).not.toHaveBeenCalled();
+ });
+
+ it("returns 400 when evaluation config is not found", async () => {
+ (getEvaluationConfig as jest.Mock).mockRejectedValue(new Error("not found"));
+
+ const response = await POST(
+ makeRequest({ types: [{ id: "hallucination", enabled: true }] }) as any
+ );
+
+ expect(response.status).toBe(400);
+ const data = await response.json();
+ expect(data.err).toBe("Evaluation config not found");
+ });
+
+ it("clamps out-of-range thresholdScore values into [0, 1] and persists them", async () => {
+ (getEvaluationConfig as jest.Mock).mockResolvedValue({
+ id: "cfg-1",
+ meta: "{}",
+ });
+
+ const response = await POST(
+ makeRequest({
+ types: [
+ { id: "toxicity", enabled: true, thresholdScore: 1.5 },
+ { id: "bias", enabled: true, thresholdScore: -0.2 },
+ ],
+ }) as any
+ );
+
+ expect(response.status).toBe(200);
+ const data = await response.json();
+ expect(data.data).toEqual([
+ expect.objectContaining({ id: "toxicity", thresholdScore: 1 }),
+ expect.objectContaining({ id: "bias", thresholdScore: 0 }),
+ ]);
+ const updateCall = (prisma.evaluationConfigs.update as jest.Mock).mock.calls[0][0];
+ expect(updateCall.where).toEqual({ id: "cfg-1" });
+ const persistedMeta = JSON.parse(updateCall.data.meta);
+ expect(persistedMeta.evaluationTypes).toEqual([
+ expect.objectContaining({ id: "toxicity", thresholdScore: 1 }),
+ expect.objectContaining({ id: "bias", thresholdScore: 0 }),
+ ]);
+ expect(syncRuleEntitiesFromConfig).toHaveBeenCalled();
+ });
+
+ it("omits thresholdScore when not provided", async () => {
+ (getEvaluationConfig as jest.Mock).mockResolvedValue({
+ id: "cfg-1",
+ meta: "{}",
+ });
+
+ const response = await POST(
+ makeRequest({ types: [{ id: "hallucination", enabled: true }] }) as any
+ );
+
+ const data = await response.json();
+ expect(data.data[0]).not.toHaveProperty("thresholdScore");
+ });
+});
diff --git a/src/client/src/__tests__/lib/platform/evaluation/evaluation-config.test.ts b/src/client/src/__tests__/lib/platform/evaluation/evaluation-config.test.ts
index 98e3169b..c20a653d 100644
--- a/src/client/src/__tests__/lib/platform/evaluation/evaluation-config.test.ts
+++ b/src/client/src/__tests__/lib/platform/evaluation/evaluation-config.test.ts
@@ -65,6 +65,7 @@ jest.mock('path', () => ({
}));
import { getEvaluationConfig, setEvaluationConfig, getEvaluationConfigById, restoreEvaluationCronJobs } from '@/lib/platform/evaluation/config';
+import { normalizeThresholdScore } from '@/lib/platform/evaluation/threshold';
import { getEvaluationTypeDefaultPrompts } from '@/lib/platform/evaluation/evaluation-type-defaults';
import { getDBConfigByUser } from '@/lib/db-config';
import prisma from '@/lib/prisma';
@@ -766,6 +767,79 @@ describe('buildEvaluationTypesWithPrompts — custom evaluation types', () => {
expect(hallucination!.rules).toHaveLength(1);
expect(hallucination!.rules![0].ruleId).toBe('rule-hal-1');
});
+
+ it('built-in type overrides carry a configured thresholdScore through', async () => {
+ const config = makeConfigWithMeta([
+ { id: 'toxicity', enabled: true, thresholdScore: 0.75 },
+ ]);
+ (asaw as jest.Mock).mockResolvedValueOnce([null, config]);
+
+ const result = await getEvaluationConfigById('eval-cfg-1');
+
+ const toxicity = result.evaluationTypes!.find((t) => t.id === 'toxicity');
+ expect(toxicity!.thresholdScore).toBe(0.75);
+ });
+
+ it('built-in types default to an undefined thresholdScore when not overridden', async () => {
+ (asaw as jest.Mock).mockResolvedValueOnce([null, makeMockEvalConfig()]);
+
+ const result = await getEvaluationConfigById('eval-cfg-1');
+
+ const hallucination = result.evaluationTypes!.find((t) => t.id === 'hallucination');
+ expect(hallucination!.thresholdScore).toBeUndefined();
+ });
+
+ it('custom types carry a configured thresholdScore through', async () => {
+ const config = makeConfigWithMeta([
+ { id: 'brand_voice', enabled: true, label: 'Brand Voice', thresholdScore: 0.3 },
+ ]);
+ (asaw as jest.Mock).mockResolvedValueOnce([null, config]);
+
+ const result = await getEvaluationConfigById('eval-cfg-1');
+
+ const brandVoice = result.evaluationTypes!.find((t) => t.id === 'brand_voice');
+ expect(brandVoice!.thresholdScore).toBe(0.3);
+ });
+
+ it('ignores a non-numeric thresholdScore stored in meta and leaves it undefined', async () => {
+ const config = makeConfigWithMeta([
+ { id: 'toxicity', enabled: true, thresholdScore: 'not-a-number' as any },
+ ]);
+ (asaw as jest.Mock).mockResolvedValueOnce([null, config]);
+
+ const result = await getEvaluationConfigById('eval-cfg-1');
+
+ const toxicity = result.evaluationTypes!.find((t) => t.id === 'toxicity');
+ expect(toxicity!.thresholdScore).toBeUndefined();
+ });
+});
+
+describe('normalizeThresholdScore', () => {
+ it('returns undefined for undefined, null, and empty string', () => {
+ expect(normalizeThresholdScore(undefined)).toBeUndefined();
+ expect(normalizeThresholdScore(null)).toBeUndefined();
+ expect(normalizeThresholdScore('')).toBeUndefined();
+ });
+
+ it('returns NaN for a non-numeric value', () => {
+ expect(Number.isNaN(normalizeThresholdScore('not-a-number'))).toBe(true);
+ });
+
+ it('passes through an in-range number unchanged', () => {
+ expect(normalizeThresholdScore(0.42)).toBe(0.42);
+ });
+
+ it('clamps values above 1 down to 1', () => {
+ expect(normalizeThresholdScore(1.5)).toBe(1);
+ });
+
+ it('clamps values below 0 up to 0', () => {
+ expect(normalizeThresholdScore(-0.5)).toBe(0);
+ });
+
+ it('accepts numeric strings', () => {
+ expect(normalizeThresholdScore('0.6')).toBe(0.6);
+ });
});
describe('restoreEvaluationCronJobs', () => {
diff --git a/src/client/src/__tests__/lib/platform/evaluation/evaluation.test.ts b/src/client/src/__tests__/lib/platform/evaluation/evaluation.test.ts
index e9260920..fca05de7 100644
--- a/src/client/src/__tests__/lib/platform/evaluation/evaluation.test.ts
+++ b/src/client/src/__tests__/lib/platform/evaluation/evaluation.test.ts
@@ -60,6 +60,8 @@ jest.mock('@/helpers/server/trace', () => ({
jest.mock('@/lib/platform/evaluation/rule-engine-context', () => ({
getContextFromRuleEngineForTrace: jest.fn().mockResolvedValue({ contextContents: [], matchingRuleIds: [], contextEntityIds: [] }),
getContextFromRulesWithPriority: jest.fn().mockResolvedValue({ contextContents: [], matchingRuleIds: [], contextEntityIds: [] }),
+ getContextFromFields: jest.fn().mockResolvedValue({ contextContents: [], matchingRuleIds: [], contextEntityIds: [] }),
+ getContextFromRulesWithPriorityForFields: jest.fn().mockResolvedValue({ contextContents: [], matchingRuleIds: [], contextEntityIds: [] }),
}));
jest.mock('@/constants/traces', () => ({
SUPPORTED_EVALUATION_OPERATIONS: ['llm', 'chat'],
@@ -75,7 +77,7 @@ jest.mock('@/lib/platform/evaluation/evaluation-type-defaults', () => ({
getEvaluationTypeDefaultPrompt: jest.fn().mockResolvedValue(undefined),
}));
-import { getEvaluationsForSpanId, getEvaluationDetectedByType, autoEvaluate, setEvaluationsForSpanId, getEvaluationSummaryForSpanId, storeManualFeedback } from '@/lib/platform/evaluation/index';
+import { getEvaluationsForSpanId, getEvaluationDetectedByType, autoEvaluate, setEvaluationsForSpanId, getEvaluationSummaryForSpanId, storeManualFeedback, runOfflineEvaluation } from '@/lib/platform/evaluation/index';
import { dataCollector } from '@/lib/platform/common';
import { getCurrentUser } from '@/lib/session';
import { getEvaluationConfig, getEvaluationConfigById } from '@/lib/platform/evaluation/config';
@@ -829,4 +831,395 @@ describe('setEvaluationsForSpanId — evaluationTypes branches', () => {
'db-1'
);
});
+
+ it('applies per-type thresholdScore when storing dashboard/manual evaluation verdicts', async () => {
+ (getRequestViaSpanId as jest.Mock).mockResolvedValue({
+ record: { SpanId: 'span-1', SpanAttributes: {} },
+ });
+ (getEvaluationConfig as jest.Mock).mockResolvedValue({
+ id: 'cfg-1',
+ provider: 'openai',
+ model: 'gpt-4',
+ secret: { value: 'sk-1' },
+ databaseConfigId: 'db-1',
+ evaluationTypes: [
+ { id: 'toxicity', enabled: true, label: 'Toxicity', thresholdScore: 0.8 },
+ ],
+ });
+ // Raw LLM verdict would be "yes" under the default 0.5 threshold; the
+ // per-type threshold of 0.8 must recompute it to "no" before storage.
+ (runEvaluation as jest.Mock).mockResolvedValue({
+ success: true,
+ result: [
+ {
+ evaluation: 'toxicity',
+ score: 0.6,
+ verdict: 'yes',
+ classification: 'mild',
+ explanation: 'borderline',
+ },
+ ],
+ });
+ (dataCollector as jest.Mock).mockResolvedValue({ data: true, err: null });
+
+ const result = await setEvaluationsForSpanId('span-1');
+ expect(result).toMatchObject({ success: true });
+
+ const insertCall = (dataCollector as jest.Mock).mock.calls.find(
+ (call) => call[1] === 'insert'
+ );
+ expect(insertCall).toBeDefined();
+ const stored = insertCall![0].values[0];
+ // storeEvaluation remaps ids to configured labels (main normalizeEvaluationResults).
+ expect(stored['evaluationData.evaluation']).toEqual(['Toxicity']);
+ expect(stored['evaluationData.verdict']).toEqual(['no']);
+ expect(stored.scores).toEqual({ Toxicity: 0.6 });
+ });
+});
+
+describe('runOfflineEvaluation', () => {
+ const baseConfig = {
+ id: 'cfg-1',
+ provider: 'openai',
+ model: 'gpt-4',
+ secret: { value: 'sk-1' },
+ databaseConfigId: 'db-1',
+ };
+
+ beforeEach(() => {
+ const { getContextFromFields, getContextFromRulesWithPriorityForFields } =
+ require('@/lib/platform/evaluation/rule-engine-context');
+ (getContextFromFields as jest.Mock).mockResolvedValue({ contextContents: [], matchingRuleIds: [], contextEntityIds: [] });
+ (getContextFromRulesWithPriorityForFields as jest.Mock).mockResolvedValue({ contextContents: [], matchingRuleIds: [], contextEntityIds: [] });
+ });
+
+ it('rejects unknown eval types without calling runEvaluation', async () => {
+ const config = {
+ ...baseConfig,
+ evaluationTypes: [{ id: 'hallucination', enabled: true, label: 'Hallucination' }],
+ };
+
+ const result = await runOfflineEvaluation(
+ { prompt: 'p', response: 'r', evalTypes: ['bogus'] },
+ config as any,
+ 'db-1'
+ );
+
+ expect(result.success).toBe(false);
+ expect(result.error).toBe('Unknown eval types: bogus');
+ expect(runEvaluation).not.toHaveBeenCalled();
+ });
+
+ it('rejects explicitly requested but disabled eval types without calling runEvaluation', async () => {
+ const config = {
+ ...baseConfig,
+ evaluationTypes: [
+ { id: 'hallucination', enabled: true, label: 'Hallucination' },
+ { id: 'bias', enabled: false, label: 'Bias' },
+ ],
+ };
+
+ const result = await runOfflineEvaluation(
+ { prompt: 'p', response: 'r', evalTypes: ['bias'] },
+ config as any,
+ 'db-1'
+ );
+
+ expect(result.success).toBe(false);
+ expect(result.error).toBe('Disabled eval types: bias');
+ expect(runEvaluation).not.toHaveBeenCalled();
+ });
+
+ it('reports unknown types even when other requested types are valid', async () => {
+ const config = {
+ ...baseConfig,
+ evaluationTypes: [{ id: 'hallucination', enabled: true, label: 'Hallucination' }],
+ };
+
+ const result = await runOfflineEvaluation(
+ { prompt: 'p', response: 'r', evalTypes: ['hallucination', 'bogus'] },
+ config as any,
+ 'db-1'
+ );
+
+ expect(result.success).toBe(false);
+ expect(result.error).toBe('Unknown eval types: bogus');
+ expect(runEvaluation).not.toHaveBeenCalled();
+ });
+
+ it('runs only the requested enabled types, excluding others from context', async () => {
+ const config = {
+ ...baseConfig,
+ evaluationTypes: [
+ { id: 'hallucination', enabled: true, label: 'Hallucination', defaultPrompt: 'hallucination ctx' },
+ { id: 'bias', enabled: true, label: 'Bias', defaultPrompt: 'bias ctx' },
+ ],
+ };
+ (runEvaluation as jest.Mock).mockResolvedValue({
+ success: true,
+ result: [{ evaluation: 'Hallucination', score: 0.1, classification: 'none', explanation: 'ok', verdict: 'no' }],
+ });
+
+ const result = await runOfflineEvaluation(
+ { prompt: 'p', response: 'r', evalTypes: ['hallucination'] },
+ config as any,
+ 'db-1'
+ );
+
+ expect(result.success).toBe(true);
+ expect(result.metadata?.evalTypesRun).toEqual(['hallucination']);
+ const [{ contexts }] = (runEvaluation as jest.Mock).mock.calls[0];
+ expect(contexts).toContain('hallucination ctx');
+ expect(contexts).not.toContain('bias ctx');
+ });
+
+ it('defaults to all enabled types when eval_types is omitted', async () => {
+ const config = {
+ ...baseConfig,
+ evaluationTypes: [
+ { id: 'hallucination', enabled: true, label: 'Hallucination' },
+ { id: 'bias', enabled: false, label: 'Bias' },
+ { id: 'relevance', enabled: true, label: 'Relevance' },
+ ],
+ };
+ (runEvaluation as jest.Mock).mockResolvedValue({ success: true, result: [] });
+
+ const result = await runOfflineEvaluation(
+ { prompt: 'p', response: 'r' },
+ config as any,
+ 'db-1'
+ );
+
+ expect(result.success).toBe(true);
+ expect([...(result.metadata?.evalTypesRun || [])].sort()).toEqual(['hallucination', 'relevance']);
+ });
+
+ it('falls back to hallucination/bias/toxicity defaults when nothing is enabled and no eval_types requested', async () => {
+ const config = {
+ ...baseConfig,
+ evaluationTypes: [
+ { id: 'hallucination', enabled: false, label: 'Hallucination' },
+ { id: 'bias', enabled: false, label: 'Bias' },
+ { id: 'toxicity', enabled: false, label: 'Toxicity' },
+ { id: 'relevance', enabled: false, label: 'Relevance' },
+ ],
+ };
+ (runEvaluation as jest.Mock).mockResolvedValue({ success: true, result: [] });
+
+ const result = await runOfflineEvaluation(
+ { prompt: 'p', response: 'r' },
+ config as any,
+ 'db-1'
+ );
+
+ expect([...(result.metadata?.evalTypesRun || [])].sort()).toEqual(['bias', 'hallucination', 'toxicity']);
+ });
+
+ it('overrides verdict to "no" when the per-type threshold is stricter than the raw score', async () => {
+ const config = {
+ ...baseConfig,
+ evaluationTypes: [{ id: 'toxicity', enabled: true, label: 'Toxicity', thresholdScore: 0.8 }],
+ };
+ // Score of 0.6 would be 'yes' under the default 0.5 threshold, but the
+ // per-type threshold of 0.8 should keep it 'no'.
+ (runEvaluation as jest.Mock).mockResolvedValue({
+ success: true,
+ result: [{ evaluation: 'Toxicity', score: 0.6, classification: 'mild', explanation: 'x', verdict: 'yes' }],
+ });
+
+ const result = await runOfflineEvaluation(
+ { prompt: 'p', response: 'r', evalTypes: ['toxicity'] },
+ config as any,
+ 'db-1'
+ );
+
+ expect(result.evaluations?.[0].verdict).toBe('no');
+ });
+
+ it('overrides verdict to "yes" when the score exceeds a stricter per-type threshold', async () => {
+ const config = {
+ ...baseConfig,
+ evaluationTypes: [{ id: 'toxicity', enabled: true, label: 'Toxicity', thresholdScore: 0.3 }],
+ };
+ (runEvaluation as jest.Mock).mockResolvedValue({
+ success: true,
+ result: [{ evaluation: 'Toxicity', score: 0.5, classification: 'mild', explanation: 'x', verdict: 'no' }],
+ });
+
+ const result = await runOfflineEvaluation(
+ { prompt: 'p', response: 'r', evalTypes: ['toxicity'] },
+ config as any,
+ 'db-1'
+ );
+
+ expect(result.evaluations?.[0].verdict).toBe('yes');
+ });
+
+ it('falls back to the request thresholdScore when the type has no per-type threshold configured', async () => {
+ const config = {
+ ...baseConfig,
+ evaluationTypes: [{ id: 'toxicity', enabled: true, label: 'Toxicity' }],
+ };
+ (runEvaluation as jest.Mock).mockResolvedValue({
+ success: true,
+ result: [{ evaluation: 'Toxicity', score: 0.9, classification: 'severe', explanation: 'x', verdict: 'no' }],
+ });
+
+ const result = await runOfflineEvaluation(
+ { prompt: 'p', response: 'r', evalTypes: ['toxicity'], thresholdScore: 0.95 },
+ config as any,
+ 'db-1'
+ );
+
+ // 0.9 does not exceed the request-level 0.95 threshold
+ expect(result.evaluations?.[0].verdict).toBe('no');
+ });
+
+ it('falls back to the request threshold when the returned evaluation label matches no configured type', async () => {
+ const config = {
+ ...baseConfig,
+ evaluationTypes: [
+ { id: 'sensitivity', enabled: true, label: 'PII - Financial', thresholdScore: 0.9 },
+ ],
+ };
+ // The model echoes back a label that doesn't match the configured id or
+ // label exactly (e.g. a custom prompt without the matching context header).
+ (runEvaluation as jest.Mock).mockResolvedValue({
+ success: true,
+ result: [{ evaluation: 'Pii Financial Data', score: 0.6, classification: 'leak', explanation: 'x', verdict: 'yes' }],
+ });
+
+ const result = await runOfflineEvaluation(
+ { prompt: 'p', response: 'r', evalTypes: ['sensitivity'], thresholdScore: 0.5 },
+ config as any,
+ 'db-1'
+ );
+
+ // The unmatched result still comes back (no crash, no dropped row), and
+ // since its label didn't match, the 0.9 per-type threshold is NOT applied —
+ // it falls back to the request-level 0.5 threshold instead.
+ expect(result.success).toBe(true);
+ expect(result.evaluations).toHaveLength(1);
+ expect(result.evaluations?.[0].verdict).toBe('yes');
+ });
+
+ it('returns failure when runEvaluation fails', async () => {
+ const config = {
+ ...baseConfig,
+ evaluationTypes: [{ id: 'hallucination', enabled: true, label: 'Hallucination' }],
+ };
+ (runEvaluation as jest.Mock).mockResolvedValue({ success: false, error: 'Model error' });
+
+ const result = await runOfflineEvaluation(
+ { prompt: 'p', response: 'r' },
+ config as any,
+ 'db-1'
+ );
+
+ expect(result.success).toBe(false);
+ expect(result.error).toBe('Model error');
+ });
+
+ describe('rule-priority context consistency with the real-time path', () => {
+ it('collects rules from enabled types and resolves context via getContextFromRulesWithPriorityForFields, matching the real-time path', async () => {
+ const { getContextFromRulesWithPriorityForFields, getContextFromFields } =
+ require('@/lib/platform/evaluation/rule-engine-context');
+ (getContextFromRulesWithPriorityForFields as jest.Mock).mockResolvedValue({
+ contextContents: ['linked rule context'],
+ matchingRuleIds: ['r1'],
+ contextEntityIds: ['e1'],
+ });
+ const config = {
+ ...baseConfig,
+ evaluationTypes: [
+ { id: 'hallucination', enabled: true, rules: [{ ruleId: 'r1', priority: 5 }] },
+ ],
+ };
+ (runEvaluation as jest.Mock).mockResolvedValue({ success: true, result: [] });
+
+ const result = await runOfflineEvaluation(
+ { prompt: 'p', response: 'r', attributes: { 'service.name': 'my-app' } },
+ config as any,
+ 'db-1'
+ );
+
+ expect(getContextFromRulesWithPriorityForFields).toHaveBeenCalledWith(
+ { 'service.name': 'my-app' },
+ [{ ruleId: 'r1', priority: 5 }],
+ 'db-1'
+ );
+ expect(getContextFromFields).not.toHaveBeenCalled();
+ expect(result.contextApplied?.matchingRuleIds).toEqual(['r1']);
+ });
+
+ it('collects legacy ruleId/priority fields into rulesWithPriority, same as the real-time path', async () => {
+ const { getContextFromRulesWithPriorityForFields } =
+ require('@/lib/platform/evaluation/rule-engine-context');
+ (getContextFromRulesWithPriorityForFields as jest.Mock).mockResolvedValue({
+ contextContents: [], matchingRuleIds: [], contextEntityIds: [],
+ });
+ const config = {
+ ...baseConfig,
+ evaluationTypes: [
+ { id: 'hallucination', enabled: true, ruleId: 'r-legacy', priority: 2 },
+ ],
+ };
+ (runEvaluation as jest.Mock).mockResolvedValue({ success: true, result: [] });
+
+ await runOfflineEvaluation(
+ { prompt: 'p', response: 'r', attributes: { 'service.name': 'my-app' } },
+ config as any,
+ 'db-1'
+ );
+
+ expect(getContextFromRulesWithPriorityForFields).toHaveBeenCalledWith(
+ { 'service.name': 'my-app' },
+ [{ ruleId: 'r-legacy', priority: 2 }],
+ 'db-1'
+ );
+ });
+
+ it('falls back to getContextFromFields when no enabled type has linked rules', async () => {
+ const { getContextFromFields, getContextFromRulesWithPriorityForFields } =
+ require('@/lib/platform/evaluation/rule-engine-context');
+ (getContextFromFields as jest.Mock).mockResolvedValue({
+ contextContents: ['generic context'], matchingRuleIds: [], contextEntityIds: [],
+ });
+ const config = {
+ ...baseConfig,
+ evaluationTypes: [{ id: 'hallucination', enabled: true }],
+ };
+ (runEvaluation as jest.Mock).mockResolvedValue({ success: true, result: [] });
+
+ await runOfflineEvaluation(
+ { prompt: 'p', response: 'r', attributes: { 'service.name': 'my-app' } },
+ config as any,
+ 'db-1'
+ );
+
+ expect(getContextFromFields).toHaveBeenCalledWith({ 'service.name': 'my-app' }, 'db-1');
+ expect(getContextFromRulesWithPriorityForFields).not.toHaveBeenCalled();
+ });
+
+ it('does not call the rule engine at all when no attributes are provided', async () => {
+ const { getContextFromFields, getContextFromRulesWithPriorityForFields } =
+ require('@/lib/platform/evaluation/rule-engine-context');
+ const config = {
+ ...baseConfig,
+ evaluationTypes: [
+ { id: 'hallucination', enabled: true, rules: [{ ruleId: 'r1', priority: 5 }] },
+ ],
+ };
+ (runEvaluation as jest.Mock).mockResolvedValue({ success: true, result: [] });
+
+ await runOfflineEvaluation(
+ { prompt: 'p', response: 'r' },
+ config as any,
+ 'db-1'
+ );
+
+ expect(getContextFromFields).not.toHaveBeenCalled();
+ expect(getContextFromRulesWithPriorityForFields).not.toHaveBeenCalled();
+ });
+ });
});
diff --git a/src/client/src/__tests__/lib/platform/evaluation/offline-auth.test.ts b/src/client/src/__tests__/lib/platform/evaluation/offline-auth.test.ts
new file mode 100644
index 00000000..037ff4e3
--- /dev/null
+++ b/src/client/src/__tests__/lib/platform/evaluation/offline-auth.test.ts
@@ -0,0 +1,152 @@
+jest.mock('@/lib/platform/api-keys', () => ({
+ getAPIKeyInfo: jest.fn(),
+}));
+jest.mock('@/lib/platform/evaluation/config', () => ({
+ getEvaluationConfigByDbConfigId: jest.fn(),
+}));
+jest.mock('@/constants/messages', () => ({
+ __esModule: true,
+ default: jest.fn(() => ({
+ NO_API_KEY: 'No API key provided',
+ })),
+}));
+jest.mock('@/utils/asaw', () =>
+ jest.fn(async (promise: Promise<any>) => {
+ try {
+ const result = await promise;
+ return [null, result];
+ } catch (err) {
+ return [err, null];
+ }
+ })
+);
+
+import {
+ authenticateOfflineApiKey,
+ loadOfflineEvaluationConfig,
+ EVALUATION_NOT_CONFIGURED_MESSAGE,
+} from '@/lib/platform/evaluation/offline-auth';
+import { getAPIKeyInfo } from '@/lib/platform/api-keys';
+import { getEvaluationConfigByDbConfigId } from '@/lib/platform/evaluation/config';
+
+class TestResponse {
+ status: number;
+ private body: unknown;
+
+ constructor(body?: unknown, init?: { status?: number }) {
+ this.body = body;
+ this.status = init?.status ?? 200;
+ }
+
+ static json(body: unknown, init?: { status?: number }) {
+ return new TestResponse(body, init);
+ }
+
+ async json() {
+ return this.body;
+ }
+}
+
+(global as any).Response = TestResponse;
+
+function makeRequest(authorization?: string) {
+ return {
+ headers: {
+ get: (key: string) => (key.toLowerCase() === 'authorization' ? authorization ?? null : null),
+ },
+ } as unknown as Request;
+}
+
+beforeEach(() => {
+ jest.clearAllMocks();
+});
+
+describe('authenticateOfflineApiKey', () => {
+ it('returns a 401 error when no Authorization header is present', async () => {
+ const result = await authenticateOfflineApiKey(makeRequest());
+ expect('error' in result).toBe(true);
+ if ('error' in result) {
+ expect(result.error.status).toBe(401);
+ }
+ expect(getAPIKeyInfo).not.toHaveBeenCalled();
+ });
+
+ it('returns a 401 error when the Authorization header is not a Bearer token', async () => {
+ const result = await authenticateOfflineApiKey(makeRequest('Basic abc123'));
+ expect('error' in result).toBe(true);
+ expect(getAPIKeyInfo).not.toHaveBeenCalled();
+ });
+
+ it('returns a 401 error when the Bearer token is empty', async () => {
+ const result = await authenticateOfflineApiKey(makeRequest('Bearer '));
+ expect('error' in result).toBe(true);
+ expect(getAPIKeyInfo).not.toHaveBeenCalled();
+ });
+
+ it('returns a 401 error when the API key is not found', async () => {
+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([new Error('not found'), null]);
+ const result = await authenticateOfflineApiKey(makeRequest('Bearer openlit-key'));
+ expect('error' in result).toBe(true);
+ if ('error' in result) {
+ expect(result.error.status).toBe(401);
+ }
+ });
+
+ it('returns a 401 error when the API key has no databaseConfigId', async () => {
+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([null, {}]);
+ const result = await authenticateOfflineApiKey(makeRequest('Bearer openlit-key'));
+ expect('error' in result).toBe(true);
+ });
+
+ it('returns the databaseConfigId for a valid API key', async () => {
+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([null, { databaseConfigId: 'db-1' }]);
+ const result = await authenticateOfflineApiKey(makeRequest('Bearer openlit-key'));
+ expect('error' in result).toBe(false);
+ if (!('error' in result)) {
+ expect(result.databaseConfigId).toBe('db-1');
+ }
+ });
+});
+
+describe('loadOfflineEvaluationConfig', () => {
+ it('calls the caller-provided notConfiguredResponse when the config lookup fails', async () => {
+ (getEvaluationConfigByDbConfigId as jest.Mock).mockRejectedValue(new Error('not found'));
+ const notConfiguredResponse = jest.fn(() => new Response(null, { status: 400 }) as any);
+
+ const result = await loadOfflineEvaluationConfig('db-1', notConfiguredResponse);
+
+ expect('error' in result).toBe(true);
+ expect(notConfiguredResponse).toHaveBeenCalledTimes(1);
+ });
+
+ it('calls the caller-provided notConfiguredResponse when the config has no id', async () => {
+ (getEvaluationConfigByDbConfigId as jest.Mock).mockResolvedValue({});
+ const notConfiguredResponse = jest.fn(() => new Response(null, { status: 400 }) as any);
+
+ const result = await loadOfflineEvaluationConfig('db-1', notConfiguredResponse);
+
+ expect('error' in result).toBe(true);
+ expect(notConfiguredResponse).toHaveBeenCalledTimes(1);
+ });
+
+ it('returns the config when found, without calling notConfiguredResponse', async () => {
+ (getEvaluationConfigByDbConfigId as jest.Mock).mockResolvedValue({ id: 'cfg-1' });
+ const notConfiguredResponse = jest.fn(() => new Response(null, { status: 400 }) as any);
+
+ const result = await loadOfflineEvaluationConfig('db-1', notConfiguredResponse);
+
+ expect('error' in result).toBe(false);
+ if (!('error' in result)) {
+ expect(result.config.id).toBe('cfg-1');
+ }
+ expect(notConfiguredResponse).not.toHaveBeenCalled();
+ expect(getEvaluationConfigByDbConfigId).toHaveBeenCalledWith('db-1', true);
+ });
+});
+
+describe('EVALUATION_NOT_CONFIGURED_MESSAGE', () => {
+ it('is a non-empty string', () => {
+ expect(typeof EVALUATION_NOT_CONFIGURED_MESSAGE).toBe('string');
+ expect(EVALUATION_NOT_CONFIGURED_MESSAGE.length).toBeGreaterThan(0);
+ });
+});
diff --git a/src/client/src/__tests__/lib/platform/evaluation/rule-engine-context.test.ts b/src/client/src/__tests__/lib/platform/evaluation/rule-engine-context.test.ts
index d1b0a7eb..2a2bd8b8 100644
--- a/src/client/src/__tests__/lib/platform/evaluation/rule-engine-context.test.ts
+++ b/src/client/src/__tests__/lib/platform/evaluation/rule-engine-context.test.ts
@@ -6,6 +6,8 @@ import {
extractRuleEngineFieldsFromTrace,
getContextFromRuleEngineForTrace,
getContextFromRulesWithPriority,
+ getContextFromFields,
+ getContextFromRulesWithPriorityForFields,
} from '@/lib/platform/evaluation/rule-engine-context';
import { evaluateRules } from '@/lib/platform/rule-engine/evaluate';
@@ -231,3 +233,105 @@ describe('getContextFromRulesWithPriority', () => {
});
});
+
+// getContextFromFields / getContextFromRulesWithPriorityForFields are the
+// shared, trace-independent primitives behind the trace-based functions
+// above — used directly by the offline/SDK evaluation path so it resolves
+// rule-engine context identically to the real-time/auto path for the same
+// input fields.
+describe('getContextFromFields', () => {
+ it('returns empty result and skips evaluateRules when fields is empty', async () => {
+ const result = await getContextFromFields({});
+ expect(result).toEqual({ contextContents: [], matchingRuleIds: [], contextEntityIds: [] });
+ expect(evaluateRules).not.toHaveBeenCalled();
+ });
+
+ it('calls evaluateRules with the given fields', async () => {
+ await getContextFromFields({ 'service.name': 'my-app' }, 'db-1');
+ expect(evaluateRules).toHaveBeenCalledWith(
+ { fields: { 'service.name': 'my-app' }, entity_type: 'context', include_entity_data: true },
+ 'db-1'
+ );
+ });
+
+ it('extracts context content from entity_data', async () => {
+ (evaluateRules as jest.Mock).mockResolvedValue({
+ matchingRuleIds: ['r1'],
+ entities: [{ rule_id: 'r1', entity_type: 'context', entity_id: 'ctx-1' }],
+ entity_data: { 'context:ctx-1': { content: 'field-based content' } },
+ });
+ const result = await getContextFromFields({ 'service.name': 'my-app' });
+ expect(result.contextContents).toEqual(['field-based content']);
+ expect(result.contextEntityIds).toEqual(['ctx-1']);
+ expect(result.matchingRuleIds).toEqual(['r1']);
+ });
+
+ it('returns empty result when evaluateRules throws', async () => {
+ (evaluateRules as jest.Mock).mockRejectedValue(new Error('boom'));
+ const result = await getContextFromFields({ 'service.name': 'my-app' });
+ expect(result).toEqual({ contextContents: [], matchingRuleIds: [], contextEntityIds: [] });
+ });
+});
+
+describe('getContextFromRulesWithPriorityForFields', () => {
+ it('returns raw result when rulesWithPriority is empty', async () => {
+ (evaluateRules as jest.Mock).mockResolvedValue({
+ matchingRuleIds: ['r1'],
+ entities: [],
+ entity_data: { 'context:ctx-1': { content: 'base content' } },
+ });
+ const result = await getContextFromRulesWithPriorityForFields({ 'service.name': 'my-app' }, []);
+ expect(result.matchingRuleIds).toContain('r1');
+ });
+
+ it('orders context by priority across the given fields, same as the trace-based path', async () => {
+ (evaluateRules as jest.Mock)
+ .mockResolvedValueOnce({ matchingRuleIds: ['r1', 'r2'], entities: [], entity_data: {} })
+ .mockResolvedValueOnce({
+ matchingRuleIds: ['r1', 'r2'],
+ entities: [
+ { rule_id: 'r1', entity_type: 'context', entity_id: 'ctx-1' },
+ { rule_id: 'r2', entity_type: 'context', entity_id: 'ctx-2' },
+ ],
+ entity_data: {
+ 'context:ctx-1': { content: 'low priority content' },
+ 'context:ctx-2': { content: 'high priority content' },
+ },
+ });
+
+ const result = await getContextFromRulesWithPriorityForFields(
+ { 'service.name': 'my-app' },
+ [
+ { ruleId: 'r1', priority: 1 },
+ { ruleId: 'r2', priority: 10 },
+ ]
+ );
+
+ expect(result.contextContents[0]).toBe('high priority content');
+ expect(result.contextContents[1]).toBe('low priority content');
+ expect(result.matchingRuleIds).toEqual(['r2', 'r1']);
+ });
+
+ it('returns empty when fields is empty (even with priority rules)', async () => {
+ const result = await getContextFromRulesWithPriorityForFields({}, [{ ruleId: 'r1', priority: 5 }]);
+ expect(result).toEqual({ contextContents: [], matchingRuleIds: [], contextEntityIds: [] });
+ });
+
+ it('returns the original (unordered) result instead of throwing when the priority re-fetch fails', async () => {
+ (evaluateRules as jest.Mock)
+ .mockResolvedValueOnce({
+ matchingRuleIds: ['r1'],
+ entities: [],
+ entity_data: { 'context:ctx-x': { content: 'fallback content' } },
+ })
+ .mockRejectedValueOnce(new Error('second fetch failed'));
+
+ const result = await getContextFromRulesWithPriorityForFields(
+ { 'service.name': 'my-app' },
+ [{ ruleId: 'r1', priority: 5 }]
+ );
+
+ expect(result.matchingRuleIds).toContain('r1');
+ expect(result.contextContents).toEqual(['fallback content']);
+ });
+});
diff --git a/src/client/src/__tests__/lib/platform/evaluation/type-config.test.ts b/src/client/src/__tests__/lib/platform/evaluation/type-config.test.ts
new file mode 100644
index 00000000..60ab0992
--- /dev/null
+++ b/src/client/src/__tests__/lib/platform/evaluation/type-config.test.ts
@@ -0,0 +1,174 @@
+jest.mock('@/lib/platform/evaluation/sync-rule-entities', () => ({
+ syncRuleEntitiesFromConfig: jest.fn().mockResolvedValue(undefined),
+}));
+jest.mock('@/utils/json', () => ({
+ jsonParse: jest.fn((v: string) => {
+ try { return JSON.parse(v); } catch { return {}; }
+ }),
+ jsonStringify: jest.fn((v: unknown) => JSON.stringify(v)),
+}));
+jest.mock('@/lib/prisma', () => ({
+ __esModule: true,
+ default: {
+ evaluationConfigs: {
+ update: jest.fn().mockResolvedValue({}),
+ },
+ },
+}));
+
+import {
+ normalizeRules,
+ normalizeTypeConfig,
+ mergeTypeIntoList,
+ upsertEvaluationTypes,
+ persistEvaluationTypes,
+} from '@/lib/platform/evaluation/type-config';
+import { syncRuleEntitiesFromConfig } from '@/lib/platform/evaluation/sync-rule-entities';
+import prisma from '@/lib/prisma';
+
+beforeEach(() => {
+ jest.clearAllMocks();
+});
+
+describe('normalizeRules', () => {
+ it('returns an empty array for a non-array input', () => {
+ expect(normalizeRules('not-an-array' as any)).toEqual([]);
+ });
+
+ it('filters out entries without a ruleId', () => {
+ const result = normalizeRules([{ ruleId: 'r1', priority: 2 }, { priority: 3 }, { ruleId: '' }]);
+ expect(result).toEqual([{ ruleId: 'r1', priority: 2 }]);
+ });
+
+ it('defaults priority to 0 when missing or non-numeric', () => {
+ const result = normalizeRules([{ ruleId: 'r1' }, { ruleId: 'r2', priority: 'bad' }]);
+ expect(result).toEqual([{ ruleId: 'r1', priority: 0 }, { ruleId: 'r2', priority: 0 }]);
+ });
+});
+
+describe('normalizeTypeConfig', () => {
+ it('builds a minimal built-in type config', () => {
+ const result = normalizeTypeConfig({ id: 'toxicity', enabled: true }, undefined);
+ expect(result).toEqual({ id: 'toxicity', enabled: true, rules: [] });
+ });
+
+ it('includes thresholdScore only when provided', () => {
+ const result = normalizeTypeConfig({ id: 'toxicity', enabled: true }, 0.8);
+ expect(result.thresholdScore).toBe(0.8);
+ const withoutThreshold = normalizeTypeConfig({ id: 'toxicity', enabled: true }, undefined);
+ expect(withoutThreshold).not.toHaveProperty('thresholdScore');
+ });
+
+ it('collects rules from the rules array over the legacy ruleId field', () => {
+ const result = normalizeTypeConfig(
+ { id: 'toxicity', enabled: true, rules: [{ ruleId: 'r1', priority: 2 }], ruleId: 'legacy' },
+ undefined
+ );
+ expect(result.rules).toEqual([{ ruleId: 'r1', priority: 2 }]);
+ });
+
+ it('falls back to the legacy ruleId/priority fields when rules is absent', () => {
+ const result = normalizeTypeConfig({ id: 'toxicity', enabled: true, ruleId: 'r-legacy', priority: 3 }, undefined);
+ expect(result.rules).toEqual([{ ruleId: 'r-legacy', priority: 3 }]);
+ });
+
+ it('preserves custom type metadata for isCustom types', () => {
+ const result = normalizeTypeConfig(
+ { id: 'my_custom', enabled: true, isCustom: true, label: ' Custom Label ', description: ' Desc ', prompt: ' Custom prompt ' },
+ undefined
+ );
+ expect(result).toMatchObject({
+ isCustom: true,
+ label: 'Custom Label',
+ description: 'Desc',
+ prompt: 'Custom prompt',
+ });
+ });
+
+ it('trims a custom prompt for built-in (non-custom) types too', () => {
+ const result = normalizeTypeConfig({ id: 'toxicity', enabled: true, prompt: ' override ' }, undefined);
+ expect(result.prompt).toBe('override');
+ expect(result.isCustom).toBeUndefined();
+ });
+});
+
+describe('mergeTypeIntoList', () => {
+ it('appends a new entry when no existing entry has the same id', () => {
+ const result = mergeTypeIntoList([{ id: 'a', enabled: true }], { id: 'b', enabled: false });
+ expect(result).toEqual([{ id: 'a', enabled: true }, { id: 'b', enabled: false }]);
+ });
+
+ it('replaces the existing entry in place when the id already exists', () => {
+ const result = mergeTypeIntoList(
+ [{ id: 'a', enabled: true }, { id: 'b', enabled: false }],
+ { id: 'a', enabled: false }
+ );
+ expect(result).toEqual([{ id: 'a', enabled: false }, { id: 'b', enabled: false }]);
+ });
+
+ it('does not mutate the original array', () => {
+ const original = [{ id: 'a', enabled: true }];
+ mergeTypeIntoList(original, { id: 'a', enabled: false });
+ expect(original).toEqual([{ id: 'a', enabled: true }]);
+ });
+});
+
+describe('upsertEvaluationTypes', () => {
+ it('upserts incoming types without dropping unrelated existing entries', () => {
+ const result = upsertEvaluationTypes(
+ [
+ { id: 'bias', enabled: true, thresholdScore: 0.4 },
+ { id: 'toxicity', enabled: false },
+ ],
+ [{ id: 'toxicity', enabled: true, thresholdScore: 0.9 }]
+ );
+ expect(result).toEqual([
+ { id: 'bias', enabled: true, thresholdScore: 0.4 },
+ { id: 'toxicity', enabled: true, thresholdScore: 0.9 },
+ ]);
+ });
+
+ it('appends types that are not already present', () => {
+ const result = upsertEvaluationTypes(
+ [{ id: 'bias', enabled: true }],
+ [{ id: 'toxicity', enabled: true }]
+ );
+ expect(result).toEqual([
+ { id: 'bias', enabled: true },
+ { id: 'toxicity', enabled: true },
+ ]);
+ });
+});
+
+describe('persistEvaluationTypes', () => {
+ it('merges evaluationTypes into the existing meta and persists it', async () => {
+ await persistEvaluationTypes(
+ 'cfg-1',
+ JSON.stringify({ cronJobId: 'keep-me' }),
+ [{ id: 'toxicity', enabled: true }]
+ );
+
+ expect(prisma.evaluationConfigs.update).toHaveBeenCalledWith({
+ where: { id: 'cfg-1' },
+ data: {
+ meta: JSON.stringify({
+ cronJobId: 'keep-me',
+ evaluationTypes: [{ id: 'toxicity', enabled: true }],
+ }),
+ },
+ });
+ });
+
+ it('handles a null/empty meta by starting fresh', async () => {
+ await persistEvaluationTypes('cfg-1', null, [{ id: 'toxicity', enabled: true }]);
+ expect(prisma.evaluationConfigs.update).toHaveBeenCalledWith({
+ where: { id: 'cfg-1' },
+ data: { meta: JSON.stringify({ evaluationTypes: [{ id: 'toxicity', enabled: true }] }) },
+ });
+ });
+
+ it('re-syncs rule entities after persisting', async () => {
+ await persistEvaluationTypes('cfg-1', '{}', []);
+ expect(syncRuleEntitiesFromConfig).toHaveBeenCalled();
+ });
+});
diff --git a/src/client/src/app/(playground)/evaluations/types/[id]/page.tsx b/src/client/src/app/(playground)/evaluations/types/[id]/page.tsx
index 43044b08..ac0ad0aa 100644
--- a/src/client/src/app/(playground)/evaluations/types/[id]/page.tsx
+++ b/src/client/src/app/(playground)/evaluations/types/[id]/page.tsx
@@ -4,6 +4,8 @@ import { useParams, useRouter } from "next/navigation";
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
import { Badge } from "@/components/ui/badge";
import { Switch } from "@/components/ui/switch";
+import { Input } from "@/components/ui/input";
+import { Label } from "@/components/ui/label";
import {
Select,
SelectContent,
@@ -38,6 +40,7 @@ interface EvaluationTypeConfig {
rules: RuleWithPriority[];
prompt?: string;
defaultPrompt?: string;
+ thresholdScore?: number;
}
export default function EvaluationTypeDetailPage() {
@@ -94,6 +97,7 @@ export default function EvaluationTypeDetailPage() {
rules: data.rules || [],
prompt: data.prompt,
defaultPrompt: data.defaultPrompt,
+ thresholdScore: data.thresholdScore,
});
return;
}
@@ -106,6 +110,7 @@ export default function EvaluationTypeDetailPage() {
rules: [],
prompt: data?.prompt,
defaultPrompt: data?.defaultPrompt,
+ thresholdScore: data?.thresholdScore,
});
return;
}
@@ -163,6 +168,7 @@ export default function EvaluationTypeDetailPage() {
enabled: config.enabled,
rules: config.rules.filter((r) => r.ruleId),
prompt: config.prompt?.trim() || undefined,
+ thresholdScore: config.thresholdScore,
};
if (config.isCustom) {
payload.isCustom = true;
@@ -313,6 +319,62 @@ export default function EvaluationTypeDetailPage() {
</Card>
)}
+ <Card className="border-stone-200 dark:border-stone-800 shadow-sm">
+ <CardHeader className="pb-2">
+ <CardTitle className="text-base">
+ {getMessage().EVALUATION_TYPE_THRESHOLD_LABEL}
+ </CardTitle>
+ <p className="text-xs text-stone-500 dark:text-stone-400 font-normal">
+ {getMessage().EVALUATION_TYPE_THRESHOLD_DESCRIPTION}
+ </p>
+ </CardHeader>
+ <CardContent>
+ <div className="max-w-[160px]">
+ <Label className="sr-only">
+ {getMessage().EVALUATION_TYPE_THRESHOLD_LABEL}
+ </Label>
+ <Input
+ type="number"
+ min={0}
+ max={1}
+ step={0.05}
+ placeholder={
+ getMessage().EVALUATION_TYPE_THRESHOLD_PLACEHOLDER
+ }
+ value={config?.thresholdScore ?? ""}
+ onChange={(e) => {
+ const raw = e.target.value;
+ setConfig((prev) => {
+ if (!prev) return prev;
+ if (raw === "") {
+ return { ...prev, thresholdScore: undefined };
+ }
+ const nextValue = Number(raw);
+ // Ignore unparseable partial input (e.g. "-", "1..2",
+ // "abc") instead of propagating NaN into config and
+ // on to the API — keep the last valid value until the
+ // user types something parseable.
+ if (Number.isNaN(nextValue)) return prev;
+ return { ...prev, thresholdScore: nextValue };
+ });
+ }}
+ onBlur={(e) => {
+ const raw = e.target.value;
+ if (raw === "") return;
+ const parsed = Number(raw);
+ if (Number.isNaN(parsed)) return;
+ const clamped = Math.min(1, Math.max(0, parsed));
+ if (clamped !== parsed) {
+ setConfig((prev) =>
+ prev ? { ...prev, thresholdScore: clamped } : prev
+ );
+ }
+ }}
+ />
+ </div>
+ </CardContent>
+ </Card>
+
<Card className="border-stone-200 dark:border-stone-800 shadow-sm">
<CardHeader className="pb-3">
<div className="flex items-start justify-between gap-4">
diff --git a/src/client/src/app/(playground)/evaluations/types/new/page.tsx b/src/client/src/app/(playground)/evaluations/types/new/page.tsx
index ef405d41..e5451a5f 100644
--- a/src/client/src/app/(playground)/evaluations/types/new/page.tsx
+++ b/src/client/src/app/(playground)/evaluations/types/new/page.tsx
@@ -69,6 +69,10 @@ export default function CreateCustomEvaluationTypePage() {
description: t.isCustom ? t.description : undefined,
prompt: t.prompt,
rules: t.rules,
+ // Preserve per-type thresholds across create-and-replace.
+ // POST /api/evaluation/types replaces the full meta list;
+ // omitting thresholdScore would wipe every saved override.
+ thresholdScore: t.thresholdScore,
})),
{
id,
diff --git a/src/client/src/app/api/evaluation/offline/types/[id]/route.ts b/src/client/src/app/api/evaluation/offline/types/[id]/route.ts
new file mode 100644
index 00000000..c046b20b
--- /dev/null
+++ b/src/client/src/app/api/evaluation/offline/types/[id]/route.ts
@@ -0,0 +1,100 @@
+import { SERVER_EVENTS } from "@/constants/events";
+import {
+ authenticateOfflineApiKey,
+ loadOfflineEvaluationConfig,
+ EVALUATION_NOT_CONFIGURED_MESSAGE,
+} from "@/lib/platform/evaluation/offline-auth";
+import { normalizeThresholdScore } from "@/lib/platform/evaluation/threshold";
+import {
+ normalizeRules,
+ mergeTypeIntoList,
+ persistEvaluationTypes,
+ EvaluationTypeConfig,
+} from "@/lib/platform/evaluation/type-config";
+import getMessage from "@/constants/messages";
+import PostHogServer from "@/lib/posthog";
+import { jsonParse } from "@/utils/json";
+import { errorResponse } from "@/helpers/server/response";
+
+export async function PATCH(
+ request: Request,
+ { params }: { params: { id: string } }
+) {
+ const startTimestamp = Date.now();
+ const typeId = params.id;
+
+ const auth = await authenticateOfflineApiKey(request);
+ if ("error" in auth) return auth.error;
+
+ let body: any;
+ try {
+ body = await request.json();
+ } catch {
+ return errorResponse("Invalid JSON body", 400);
+ }
+
+ const thresholdScore =
+ body.thresholdScore !== undefined
+ ? normalizeThresholdScore(body.thresholdScore)
+ : undefined;
+ if (Number.isNaN(thresholdScore)) {
+ PostHogServer.fireEvent({
+ event: SERVER_EVENTS.EVALUATION_OFFLINE_TYPE_UPDATE_FAILURE,
+ startTimestamp,
+ });
+ return errorResponse(getMessage().EVALUATION_THRESHOLD_SCORE_INVALID, 400);
+ }
+
+ const loaded = await loadOfflineEvaluationConfig(auth.databaseConfigId, () => {
+ PostHogServer.fireEvent({
+ event: SERVER_EVENTS.EVALUATION_OFFLINE_TYPE_UPDATE_FAILURE,
+ startTimestamp,
+ });
+ return errorResponse(EVALUATION_NOT_CONFIGURED_MESSAGE, 400);
+ });
+ if ("error" in loaded) return loaded.error;
+
+ const meta = jsonParse((loaded.config as any).meta || "{}") as Record<string, any>;
+ const types: EvaluationTypeConfig[] =
+ (meta.evaluationTypes as EvaluationTypeConfig[]) || [];
+ const idx = types.findIndex((t: any) => t.id === typeId);
+ const existing = idx >= 0 ? types[idx] : { id: typeId, enabled: false, rules: [] };
+ const updated: EvaluationTypeConfig = {
+ id: typeId,
+ enabled: body.enabled ?? existing.enabled,
+ rules: body.rules !== undefined ? normalizeRules(body.rules) : existing.rules || [],
+ prompt: body.prompt !== undefined ? body.prompt : existing.prompt,
+ thresholdScore:
+ body.thresholdScore !== undefined
+ ? thresholdScore
+ : (existing as any).thresholdScore,
+ };
+ if (body.isCustom || (existing as any).isCustom) {
+ updated.isCustom = true;
+ updated.label = body.label ?? (existing as any).label ?? typeId;
+ updated.description = body.description ?? (existing as any).description ?? "";
+ }
+
+ await persistEvaluationTypes(
+ loaded.config.id,
+ (loaded.config as any).meta,
+ mergeTypeIntoList(types, updated)
+ );
+
+ PostHogServer.fireEvent({
+ event: SERVER_EVENTS.EVALUATION_OFFLINE_TYPE_UPDATE_SUCCESS,
+ startTimestamp,
+ });
+ return Response.json({ data: updated });
+}
+
+export async function OPTIONS() {
+ return new Response(null, {
+ status: 200,
+ headers: {
+ "Access-Control-Allow-Origin": "*",
+ "Access-Control-Allow-Methods": "PATCH, OPTIONS",
+ "Access-Control-Allow-Headers": "Content-Type, Authorization",
+ },
+ });
+}
diff --git a/src/client/src/app/api/evaluation/offline/types/route.ts b/src/client/src/app/api/evaluation/offline/types/route.ts
index fc038166..afbaec87 100644
--- a/src/client/src/app/api/evaluation/offline/types/route.ts
+++ b/src/client/src/app/api/evaluation/offline/types/route.ts
@@ -1,51 +1,130 @@
-import { getAPIKeyInfo } from "@/lib/platform/api-keys";
-import { getEvaluationConfigByDbConfigId } from "@/lib/platform/evaluation/config";
+import { SERVER_EVENTS } from "@/constants/events";
+import {
+ authenticateOfflineApiKey,
+ loadOfflineEvaluationConfig,
+ EVALUATION_NOT_CONFIGURED_MESSAGE,
+} from "@/lib/platform/evaluation/offline-auth";
+import { normalizeThresholdScore } from "@/lib/platform/evaluation/threshold";
+import {
+ normalizeTypeConfig,
+ persistEvaluationTypes,
+ upsertEvaluationTypes,
+ EvaluationTypeConfig,
+} from "@/lib/platform/evaluation/type-config";
import getMessage from "@/constants/messages";
-import asaw from "@/utils/asaw";
+import PostHogServer from "@/lib/posthog";
import { errorResponse } from "@/helpers/server/response";
+import { jsonParse } from "@/utils/json";
export async function GET(request: Request) {
- const authorizationHeader = request.headers.get("Authorization") || "";
- if (!authorizationHeader.startsWith("Bearer ")) {
- return errorResponse(getMessage().NO_API_KEY, 401);
- }
- const apiKey = authorizationHeader.replace(/^Bearer /, "").trim();
- if (!apiKey) {
- return errorResponse(getMessage().NO_API_KEY, 401);
- }
- const [keyErr, apiInfo] = await getAPIKeyInfo({ apiKey });
- if (keyErr || !apiInfo?.databaseConfigId) {
- return errorResponse(getMessage().NO_API_KEY, 401);
- }
+ const startTimestamp = Date.now();
- const [configErr, config] = await asaw(
- getEvaluationConfigByDbConfigId(apiInfo.databaseConfigId, true)
+ const auth = await authenticateOfflineApiKey(request);
+ if ("error" in auth) return auth.error;
+
+ const loaded = await loadOfflineEvaluationConfig(auth.databaseConfigId, () =>
+ errorResponse(EVALUATION_NOT_CONFIGURED_MESSAGE, 200, {
+ eval_types: [],
+ configured: false,
+ })
);
- if (configErr || !config?.id) {
- return errorResponse(
- "Evaluation not configured. Set up evaluation in the OpenLIT dashboard first.",
- 200,
- { eval_types: [], configured: false }
- );
- }
+ if ("error" in loaded) return loaded.error;
- const types = ((config as any).evaluationTypes || []).map((t: any) => ({
+ const types = (loaded.config.evaluationTypes || []).map((t: any) => ({
id: t.id,
label: t.label || t.id,
description: t.description || "",
enabled: !!t.enabled,
is_custom: !!t.isCustom,
+ ...(typeof t.thresholdScore === "number"
+ ? { threshold_score: t.thresholdScore }
+ : {}),
}));
+ PostHogServer.fireEvent({
+ event: SERVER_EVENTS.EVALUATION_OFFLINE_TYPES_SUCCESS,
+ startTimestamp,
+ });
return Response.json({ eval_types: types });
}
+export async function POST(request: Request) {
+ const startTimestamp = Date.now();
+
+ const auth = await authenticateOfflineApiKey(request);
+ if ("error" in auth) return auth.error;
+
+ let body: any;
+ try {
+ body = await request.json();
+ } catch {
+ return errorResponse("Invalid JSON body", 400);
+ }
+ const types = body.types as any[] | undefined;
+ if (!Array.isArray(types)) {
+ PostHogServer.fireEvent({
+ event: SERVER_EVENTS.EVALUATION_OFFLINE_TYPE_CREATE_FAILURE,
+ startTimestamp,
+ });
+ return errorResponse("Invalid types array", 400);
+ }
+
+ const thresholdScores: Array<number | undefined> = [];
+ for (const t of types) {
+ const normalized = normalizeThresholdScore(t?.thresholdScore);
+ if (Number.isNaN(normalized)) {
+ PostHogServer.fireEvent({
+ event: SERVER_EVENTS.EVALUATION_OFFLINE_TYPE_CREATE_FAILURE,
+ startTimestamp,
+ });
+ return errorResponse(getMessage().EVALUATION_THRESHOLD_SCORE_INVALID, 400);
+ }
+ thresholdScores.push(normalized);
+ }
+
+ const loaded = await loadOfflineEvaluationConfig(auth.databaseConfigId, () => {
+ PostHogServer.fireEvent({
+ event: SERVER_EVENTS.EVALUATION_OFFLINE_TYPE_CREATE_FAILURE,
+ startTimestamp,
+ });
+ return errorResponse(EVALUATION_NOT_CONFIGURED_MESSAGE, 400);
+ });
+ if ("error" in loaded) return loaded.error;
+
+ const normalizedTypes = types.map((t, i) =>
+ normalizeTypeConfig(t, thresholdScores[i])
+ );
+
+ // Upsert into the existing meta list. Unlike the dashboard POST (which
+ // always sends the full UI list), API-key clients may post a partial
+ // array — a full replace would wipe unrelated types and thresholds.
+ const meta = jsonParse((loaded.config as any).meta || "{}") as Record<
+ string,
+ any
+ >;
+ const existingTypes: EvaluationTypeConfig[] =
+ (meta.evaluationTypes as EvaluationTypeConfig[]) || [];
+ const nextTypes = upsertEvaluationTypes(existingTypes, normalizedTypes);
+
+ await persistEvaluationTypes(
+ loaded.config.id,
+ (loaded.config as any).meta,
+ nextTypes
+ );
+
+ PostHogServer.fireEvent({
+ event: SERVER_EVENTS.EVALUATION_OFFLINE_TYPE_CREATE_SUCCESS,
+ startTimestamp,
+ });
+ return Response.json({ data: normalizedTypes });
+}
+
export async function OPTIONS() {
return new Response(null, {
status: 200,
headers: {
"Access-Control-Allow-Origin": "*",
- "Access-Control-Allow-Methods": "GET, OPTIONS",
+ "Access-Control-Allow-Methods": "GET, POST, OPTIONS",
"Access-Control-Allow-Headers": "Content-Type, Authorization",
},
});
diff --git a/src/client/src/app/api/evaluation/types/[id]/route.ts b/src/client/src/app/api/evaluation/types/[id]/route.ts
index f10d5e98..8664af18 100644
--- a/src/client/src/app/api/evaluation/types/[id]/route.ts
+++ b/src/client/src/app/api/evaluation/types/[id]/route.ts
@@ -1,38 +1,24 @@
import { withAudit } from "@/lib/audit/route";
import { withCurrentOrganisationPermission } from "@/lib/rbac/current";
import { getEvaluationConfig } from "@/lib/platform/evaluation/config";
-import { syncRuleEntitiesFromConfig } from "@/lib/platform/evaluation/sync-rule-entities";
+import { normalizeThresholdScore } from "@/lib/platform/evaluation/threshold";
+import {
+ normalizeRules,
+ mergeTypeIntoList,
+ persistEvaluationTypes,
+ EvaluationTypeConfig,
+} from "@/lib/platform/evaluation/type-config";
import { SERVER_EVENTS } from "@/constants/events";
import PostHogServer from "@/lib/posthog";
import { NextRequest } from "next/server";
import asaw from "@/utils/asaw";
-import { jsonParse, jsonStringify } from "@/utils/json";
+import { jsonParse } from "@/utils/json";
+import getMessage from "@/constants/messages";
-export interface RuleWithPriority {
- ruleId: string;
- priority: number;
-}
-
-export interface EvaluationTypeConfig {
- id: string;
- enabled: boolean;
- isCustom?: boolean;
- label?: string;
- description?: string;
- rules?: RuleWithPriority[];
- prompt?: string;
- defaultPrompt?: string;
-}
-
-function normalizeRules(rules: any[]): RuleWithPriority[] {
- if (!Array.isArray(rules)) return [];
- return rules
- .filter((r: any) => r?.ruleId)
- .map((r: any) => ({
- ruleId: r.ruleId,
- priority: Number(r.priority) || 0,
- }));
-}
+export type {
+ RuleWithPriority,
+ EvaluationTypeConfig,
+} from "@/lib/platform/evaluation/type-config";
async function GETHandler(
_: NextRequest,
@@ -63,7 +49,28 @@ async function PATCHHandler(
) {
const startTimestamp = Date.now();
const typeId = params.id;
- const body = await request.json();
+ let body: any;
+ try {
+ body = await request.json();
+ } catch {
+ return Response.json({ err: "Invalid JSON body" }, { status: 400 });
+ }
+
+ const thresholdScore =
+ body.thresholdScore !== undefined
+ ? normalizeThresholdScore(body.thresholdScore)
+ : undefined;
+ if (Number.isNaN(thresholdScore)) {
+ PostHogServer.fireEvent({
+ event: SERVER_EVENTS.EVALUATION_TYPE_UPDATE_FAILURE,
+ startTimestamp,
+ });
+ return Response.json(
+ { err: getMessage().EVALUATION_THRESHOLD_SCORE_INVALID },
+ { status: 400 }
+ );
+ }
+
const [err, config] = await asaw(getEvaluationConfig(undefined, true, false));
if (err || !config?.id) {
PostHogServer.fireEvent({
@@ -75,9 +82,8 @@ async function PATCHHandler(
{ status: 400 }
);
}
- const prisma = (await import("@/lib/prisma")).default;
const meta = jsonParse((config as any).meta || "{}") as Record<string, any>;
- let types: EvaluationTypeConfig[] =
+ const types: EvaluationTypeConfig[] =
(meta.evaluationTypes as EvaluationTypeConfig[]) || [];
const idx = types.findIndex((t: any) => t.id === typeId);
const existing = idx >= 0 ? types[idx] : { id: typeId, enabled: false, rules: [] };
@@ -86,6 +92,10 @@ async function PATCHHandler(
enabled: body.enabled ?? existing.enabled,
rules: body.rules !== undefined ? normalizeRules(body.rules) : existing.rules || [],
prompt: body.prompt !== undefined ? body.prompt : existing.prompt,
+ thresholdScore:
+ body.thresholdScore !== undefined
+ ? thresholdScore
+ : (existing as any).thresholdScore,
};
// Preserve or update custom type metadata
if (body.isCustom || (existing as any).isCustom) {
@@ -93,17 +103,13 @@ async function PATCHHandler(
updated.label = body.label ?? (existing as any).label ?? typeId;
updated.description = body.description ?? (existing as any).description ?? "";
}
- if (idx >= 0) {
- types[idx] = updated;
- } else {
- types = [...types, updated];
- }
- meta.evaluationTypes = types;
- await prisma.evaluationConfigs.update({
- where: { id: config.id },
- data: { meta: jsonStringify(meta) },
- });
- await syncRuleEntitiesFromConfig();
+
+ await persistEvaluationTypes(
+ config.id,
+ (config as any).meta,
+ mergeTypeIntoList(types, updated)
+ );
+
PostHogServer.fireEvent({
event: SERVER_EVENTS.EVALUATION_TYPE_UPDATE_SUCCESS,
startTimestamp,
@@ -128,7 +134,6 @@ async function DELETEHandler(
{ status: 400 }
);
}
- const prisma = (await import("@/lib/prisma")).default;
const meta = jsonParse((config as any).meta || "{}") as Record<string, any>;
const types: EvaluationTypeConfig[] =
(meta.evaluationTypes as EvaluationTypeConfig[]) || [];
@@ -145,12 +150,12 @@ async function DELETEHandler(
);
}
- meta.evaluationTypes = types.filter((t) => t.id !== typeId);
- await prisma.evaluationConfigs.update({
- where: { id: config.id },
- data: { meta: jsonStringify(meta) },
- });
- await syncRuleEntitiesFromConfig();
+ await persistEvaluationTypes(
+ config.id,
+ (config as any).meta,
+ types.filter((t) => t.id !== typeId)
+ );
+
PostHogServer.fireEvent({
event: SERVER_EVENTS.EVALUATION_TYPE_DELETE_SUCCESS,
startTimestamp,
diff --git a/src/client/src/app/api/evaluation/types/route.ts b/src/client/src/app/api/evaluation/types/route.ts
index 19592e5a..0fba22d7 100644
--- a/src/client/src/app/api/evaluation/types/route.ts
+++ b/src/client/src/app/api/evaluation/types/route.ts
@@ -1,56 +1,21 @@
import { withAudit } from "@/lib/audit/route";
import { withCurrentOrganisationPermission } from "@/lib/rbac/current";
import { getEvaluationConfig } from "@/lib/platform/evaluation/config";
-import { syncRuleEntitiesFromConfig } from "@/lib/platform/evaluation/sync-rule-entities";
+import { normalizeThresholdScore } from "@/lib/platform/evaluation/threshold";
+import {
+ normalizeTypeConfig,
+ persistEvaluationTypes,
+} from "@/lib/platform/evaluation/type-config";
import { SERVER_EVENTS } from "@/constants/events";
import PostHogServer from "@/lib/posthog";
import { NextRequest } from "next/server";
import asaw from "@/utils/asaw";
-import { jsonParse, jsonStringify } from "@/utils/json";
+import getMessage from "@/constants/messages";
-export interface RuleWithPriority {
- ruleId: string;
- priority: number;
-}
-
-export interface EvaluationTypeConfig {
- id: string;
- enabled: boolean;
- isCustom?: boolean;
- label?: string;
- description?: string;
- rules?: RuleWithPriority[];
- ruleId?: string;
- priority?: number;
- defaultPrompt?: string;
- prompt?: string;
-}
-
-function normalizeTypeConfig(t: any): EvaluationTypeConfig {
- const rules = t.rules?.length
- ? t.rules.filter((r: any) => r?.ruleId).map((r: any) => ({
- ruleId: r.ruleId,
- priority: Number(r.priority) || 0,
- }))
- : t.ruleId
- ? [{ ruleId: t.ruleId, priority: Number(t.priority) || 0 }]
- : [];
- const config: EvaluationTypeConfig = {
- id: t.id,
- enabled: !!t.enabled,
- rules,
- };
- // Preserve custom type metadata
- if (t.isCustom) {
- config.isCustom = true;
- if (t.label) config.label = String(t.label).trim();
- if (t.description) config.description = String(t.description).trim();
- if (t.prompt) config.prompt = String(t.prompt).trim();
- } else if (t.prompt) {
- config.prompt = String(t.prompt).trim();
- }
- return config;
-}
+export type {
+ RuleWithPriority,
+ EvaluationTypeConfig,
+} from "@/lib/platform/evaluation/type-config";
async function GETHandler(_: NextRequest) {
const [err, config] = await asaw(getEvaluationConfig(undefined, true, false));
@@ -67,7 +32,12 @@ async function GETHandler(_: NextRequest) {
async function POSTHandler(request: NextRequest) {
const startTimestamp = Date.now();
- const body = await request.json();
+ let body: any;
+ try {
+ body = await request.json();
+ } catch {
+ return Response.json({ err: "Invalid JSON body" }, { status: 400 });
+ }
const types = body.types as any[] | undefined;
if (!Array.isArray(types)) {
PostHogServer.fireEvent({
@@ -77,6 +47,22 @@ async function POSTHandler(request: NextRequest) {
return Response.json({ err: "Invalid types array" }, { status: 400 });
}
+ const thresholdScores: Array<number | undefined> = [];
+ for (const t of types) {
+ const normalized = normalizeThresholdScore(t?.thresholdScore);
+ if (Number.isNaN(normalized)) {
+ PostHogServer.fireEvent({
+ event: SERVER_EVENTS.EVALUATION_TYPE_CREATE_FAILURE,
+ startTimestamp,
+ });
+ return Response.json(
+ { err: getMessage().EVALUATION_THRESHOLD_SCORE_INVALID },
+ { status: 400 }
+ );
+ }
+ thresholdScores.push(normalized);
+ }
+
const [err, config] = await asaw(getEvaluationConfig(undefined, true, false));
if (err || !config?.id) {
PostHogServer.fireEvent({
@@ -86,18 +72,11 @@ async function POSTHandler(request: NextRequest) {
return Response.json({ err: "Evaluation config not found" }, { status: 400 });
}
- const normalizedTypes = types.map(normalizeTypeConfig);
-
- const prisma = (await import("@/lib/prisma")).default;
- const meta = jsonParse((config as any).meta || "{}") as Record<string, any>;
- meta.evaluationTypes = normalizedTypes;
-
- await prisma.evaluationConfigs.update({
- where: { id: config.id },
- data: { meta: jsonStringify(meta) },
- });
+ const normalizedTypes = types.map((t, i) =>
+ normalizeTypeConfig(t, thresholdScores[i])
+ );
- await syncRuleEntitiesFromConfig();
+ await persistEvaluationTypes(config.id, (config as any).meta, normalizedTypes);
PostHogServer.fireEvent({
event: SERVER_EVENTS.EVALUATION_TYPE_CREATE_SUCCESS,
diff --git a/src/client/src/constants/events.ts b/src/client/src/constants/events.ts
index 80150293..c8f056fc 100644
--- a/src/client/src/constants/events.ts
+++ b/src/client/src/constants/events.ts
@@ -259,6 +259,12 @@ export const SERVER_EVENTS = {
EVALUATION_LLM_RUN_FAILURE: "EVALUATION_LLM_RUN_FAILURE",
EVALUATION_OFFLINE_SUCCESS: "EVALUATION_OFFLINE_SUCCESS",
EVALUATION_OFFLINE_FAILURE: "EVALUATION_OFFLINE_FAILURE",
+ EVALUATION_OFFLINE_TYPES_SUCCESS: "EVALUATION_OFFLINE_TYPES_SUCCESS",
+ EVALUATION_OFFLINE_TYPES_FAILURE: "EVALUATION_OFFLINE_TYPES_FAILURE",
+ EVALUATION_OFFLINE_TYPE_CREATE_SUCCESS: "EVALUATION_OFFLINE_TYPE_CREATE_SUCCESS",
+ EVALUATION_OFFLINE_TYPE_CREATE_FAILURE: "EVALUATION_OFFLINE_TYPE_CREATE_FAILURE",
+ EVALUATION_OFFLINE_TYPE_UPDATE_SUCCESS: "EVALUATION_OFFLINE_TYPE_UPDATE_SUCCESS",
+ EVALUATION_OFFLINE_TYPE_UPDATE_FAILURE: "EVALUATION_OFFLINE_TYPE_UPDATE_FAILURE",
// User
USER_ONBOARDING_COMPLETE_SUCCESS: "USER_ONBOARDING_COMPLETE_SUCCESS",
diff --git a/src/client/src/constants/messages/en.ts b/src/client/src/constants/messages/en.ts
index 91fef011..5dd2b0bf 100644
--- a/src/client/src/constants/messages/en.ts
+++ b/src/client/src/constants/messages/en.ts
@@ -115,6 +115,12 @@ export const EVALUATION_SAMPLE_RATE_DESCRIPTION =
"Percentage of traces to auto-evaluate. Uses deterministic sampling per trace to reduce LLM cost. Traces not selected are marked as skipped and are not re-queried on future cron runs; manual evaluation is always available.";
export const EVALUATION_SAMPLE_RATE_INVALID =
"Sample rate must be a number between 0 and 1.";
+export const EVALUATION_THRESHOLD_SCORE_INVALID =
+ "Threshold score must be a number between 0 and 1.";
+export const EVALUATION_TYPE_THRESHOLD_LABEL = "Verdict threshold";
+export const EVALUATION_TYPE_THRESHOLD_DESCRIPTION =
+ "Score above which this type's verdict is flagged as \"yes\". Applies to dashboard, auto, and offline evaluations. Leave blank to use the default threshold (0.5).";
+export const EVALUATION_TYPE_THRESHOLD_PLACEHOLDER = "0.5 (default)";
export const EVALUATION_SAMPLE_RATE_PERCENT_INVALID =
"Sample rate must be a number between 0 and 100.";
export const EVALUATION_SAVING = "Saving...";
diff --git a/src/client/src/lib/platform/evaluation/config.ts b/src/client/src/lib/platform/evaluation/config.ts
index ccf99f78..c3550798 100644
--- a/src/client/src/lib/platform/evaluation/config.ts
+++ b/src/client/src/lib/platform/evaluation/config.ts
@@ -47,6 +47,7 @@ export interface EvaluationTypeWithPrompt {
rules?: Array<{ ruleId: string; priority: number }>;
prompt?: string;
defaultPrompt: string;
+ thresholdScore?: number;
}
async function buildEvaluationTypesWithPrompts(
@@ -61,6 +62,7 @@ async function buildEvaluationTypesWithPrompts(
isCustom?: boolean;
rules?: Array<{ ruleId: string; priority: number }>;
prompt?: string;
+ thresholdScore?: number;
}>) || [];
const overrideMap = new Map(
@@ -82,6 +84,10 @@ async function buildEvaluationTypesWithPrompts(
rules: override?.rules?.filter((r) => r?.ruleId) ?? [],
prompt: override?.prompt,
defaultPrompt: defaultPrompts[et.id] ?? "",
+ thresholdScore:
+ typeof override?.thresholdScore === "number"
+ ? override.thresholdScore
+ : undefined,
};
});
@@ -98,6 +104,8 @@ async function buildEvaluationTypesWithPrompts(
rules: t.rules?.filter((r) => r?.ruleId) ?? [],
prompt: t.prompt,
defaultPrompt: "",
+ thresholdScore:
+ typeof t.thresholdScore === "number" ? t.thresholdScore : undefined,
}));
return [...builtInTypes, ...customTypes];
diff --git a/src/client/src/lib/platform/evaluation/index.ts b/src/client/src/lib/platform/evaluation/index.ts
index 71290c5a..7a2aebfb 100644
--- a/src/client/src/lib/platform/evaluation/index.ts
+++ b/src/client/src/lib/platform/evaluation/index.ts
@@ -7,7 +7,6 @@ import { OPENLIT_EVALUATION_TABLE_NAME } from "./table-details";
import { runEvaluation } from "./run-evaluation";
import { getEvaluationConfig, getEvaluationConfigById } from "./config";
import asaw from "@/utils/asaw";
-import { evaluateRules } from "@/lib/platform/rule-engine/evaluate";
import {
AutoEvaluationConfig,
Evaluation,
@@ -21,6 +20,8 @@ import { getTraceMappingKeyFullPath } from "@/helpers/server/trace";
import {
getContextFromRuleEngineForTrace,
getContextFromRulesWithPriority,
+ getContextFromFields,
+ getContextFromRulesWithPriorityForFields,
} from "./rule-engine-context";
import { TraceRow } from "@/types/trace";
import { get } from "lodash";
@@ -50,6 +51,56 @@ import {
type EvaluationTypeRef,
} from "@/helpers/client/evaluation-type";
+/**
+ * Collects rule links (with priority) across a set of enabled evaluation
+ * types. Shared by the real-time/auto and offline evaluation paths so both
+ * resolve rule-engine context identically for the same configured types.
+ */
+function collectRulesWithPriority(
+ types: Array<{
+ rules?: Array<{ ruleId: string; priority: number }>;
+ ruleId?: string;
+ priority?: number;
+ }>
+): Array<{ ruleId: string; priority: number }> {
+ const rulesWithPriority: Array<{ ruleId: string; priority: number }> = [];
+ for (const t of types) {
+ if (t.rules?.length) {
+ rulesWithPriority.push(
+ ...t.rules.filter((r) => r.ruleId && r.ruleId.trim())
+ );
+ } else if (t.ruleId) {
+ rulesWithPriority.push({
+ ruleId: t.ruleId,
+ priority: t.priority ?? 0,
+ });
+ }
+ }
+ return rulesWithPriority;
+}
+
+function applyPerTypeThresholds(
+ results: Evaluation[],
+ enabledTypes: Array<{ id: string; label?: string; thresholdScore?: number }>,
+ fallbackThreshold: number
+): Evaluation[] {
+ return results.map((evaluation) => {
+ const type = enabledTypes.find(
+ (t) =>
+ t.id.toLowerCase() === evaluation.evaluation.toLowerCase() ||
+ t.label?.toLowerCase() === evaluation.evaluation.toLowerCase()
+ );
+ const threshold =
+ typeof type?.thresholdScore === "number"
+ ? type.thresholdScore
+ : fallbackThreshold;
+ return {
+ ...evaluation,
+ verdict: evaluation.score > threshold ? "yes" : "no",
+ };
+ });
+}
+
/**
* Cost from Manage Models (`openlit_provider_models`) — same source as
* Pricing / Openground. Returns 0 if the model is not configured.
@@ -475,15 +526,18 @@ async function getEvaluationConfigForTrace(
const prompt = get(trace, getTraceMappingKeyFullPath("prompt", true));
const dbConfig = dbConfigId ?? evaluationConfig.databaseConfigId;
+ const DEFAULT_THRESHOLD = 0.5;
const evaluationTypes =
((evaluationConfig as any).evaluationTypes || []) as Array<{
id: string;
enabled: boolean;
+ label?: string;
rules?: Array<{ ruleId: string; priority: number }>;
ruleId?: string;
priority?: number;
prompt?: string;
defaultPrompt?: string;
+ thresholdScore?: number;
}>;
// Default: Hallucination, Bias, Toxicity enabled
@@ -498,19 +552,7 @@ async function getEvaluationConfigForTrace(
).map((t) => ({ ...t, enabled: true }));
// Collect rules with priority from enabled types
- const rulesWithPriority: Array<{ ruleId: string; priority: number }> = [];
- for (const t of enabledTypes) {
- if (t.rules?.length) {
- rulesWithPriority.push(
- ...t.rules.filter((r) => r.ruleId && r.ruleId.trim())
- );
- } else if (t.ruleId) {
- rulesWithPriority.push({
- ruleId: t.ruleId,
- priority: t.priority ?? 0,
- });
- }
- }
+ const rulesWithPriority = collectRulesWithPriority(enabledTypes);
let contextContents: string[];
let matchingRuleIds: string[];
@@ -553,7 +595,7 @@ async function getEvaluationConfigForTrace(
prompt: prompt ?? "",
contexts: finalContextString,
response: response ?? "",
- thresholdScore: 0.5,
+ thresholdScore: DEFAULT_THRESHOLD,
});
@@ -561,6 +603,14 @@ async function getEvaluationConfigForTrace(
return { success: false, error: data.error };
}
+ // Recompute verdicts with per-type thresholds, matching the offline
+ // path so dashboard/auto/manual and SDK runs honor the same config.
+ const evaluations = applyPerTypeThresholds(
+ data.result || [],
+ enabledTypes,
+ DEFAULT_THRESHOLD
+ );
+
const metaBase: Record<string, string> = {
model: `${evaluationConfig.provider}/${evaluationConfig.model}`,
traceTimeStamp: String(trace.Timestamp ?? ""),
@@ -584,7 +634,7 @@ async function getEvaluationConfigForTrace(
const storeResult = await storeEvaluation(
trace.SpanId,
- data.result || [],
+ evaluations,
metaBase,
dbConfig,
evaluationTypes.map((t) => ({ id: t.id, label: (t as any).label || t.id }))
@@ -867,17 +917,10 @@ export async function runOfflineEvaluation(
priority?: number;
prompt?: string;
defaultPrompt?: string;
+ thresholdScore?: number;
}>;
- let enabledTypes = requestedTypes?.length
- ? allTypes.filter((t) => requestedTypes.includes(t.id))
- : allTypes.filter((t) => t.enabled);
-
- if (enabledTypes.length === 0) {
- enabledTypes = allTypes
- .filter((t) => ["hallucination", "bias", "toxicity"].includes(t.id))
- .map((t) => ({ ...t, enabled: true }));
- }
+ let enabledTypes: typeof allTypes;
if (requestedTypes?.length) {
const allTypeIds = new Set(allTypes.map((t) => t.id));
@@ -888,39 +931,50 @@ export async function runOfflineEvaluation(
error: `Unknown eval types: ${unknown.join(", ")}`,
};
}
+
+ const disabled = requestedTypes.filter((id) => {
+ const type = allTypes.find((t) => t.id === id);
+ return type && !type.enabled;
+ });
+ if (disabled.length > 0) {
+ return {
+ success: false,
+ error: `Disabled eval types: ${disabled.join(", ")}`,
+ };
+ }
+
+ enabledTypes = allTypes.filter((t) => requestedTypes.includes(t.id));
+ } else {
+ enabledTypes = allTypes.filter((t) => t.enabled);
+
+ if (enabledTypes.length === 0) {
+ enabledTypes = allTypes
+ .filter((t) => ["hallucination", "bias", "toxicity"].includes(t.id))
+ .map((t) => ({ ...t, enabled: true }));
+ }
}
+ // Collect rules with priority from enabled types, same as the real-time/
+ // auto path, so offline evals resolve rule-engine context identically for
+ // the same configured evaluation types.
+ const rulesWithPriority = collectRulesWithPriority(enabledTypes);
+
let contextContents: string[] = [];
let matchingRuleIds: string[] = [];
let contextEntityIds: string[] = [];
if (attributes && Object.keys(attributes).length > 0) {
- try {
- const ruleResult = await evaluateRules(
- {
- fields: attributes,
- entity_type: "context" as any,
- include_entity_data: true,
- },
- databaseConfigId
- );
-
- matchingRuleIds = ruleResult.matchingRuleIds || [];
-
- if (ruleResult.entity_data) {
- for (const [key, entityData] of Object.entries(
- ruleResult.entity_data
- )) {
- if (entityData?.content) {
- contextContents.push(String(entityData.content));
- const match = key.match(/^context:(.+)$/);
- if (match) contextEntityIds.push(match[1]);
- }
- }
- }
- } catch (e) {
- consoleLog("Rule engine evaluation failed for offline eval:", e);
- }
+ const result =
+ rulesWithPriority.length > 0
+ ? await getContextFromRulesWithPriorityForFields(
+ attributes,
+ rulesWithPriority,
+ databaseConfigId
+ )
+ : await getContextFromFields(attributes, databaseConfigId);
+ contextContents = result.contextContents;
+ matchingRuleIds = result.matchingRuleIds;
+ contextEntityIds = result.contextEntityIds || [];
}
const userContextsCount = userContexts?.length || 0;
@@ -952,6 +1006,12 @@ export async function runOfflineEvaluation(
return { success: false, error: data.error };
}
+ const evaluations = applyPerTypeThresholds(
+ data.result || [],
+ enabledTypes,
+ thresholdScore
+ );
+
const metaBase: Record<string, string> = {
model: `${evaluationConfig.provider}/${evaluationConfig.model}`,
ruleIds: matchingRuleIds.join(","),
@@ -988,7 +1048,7 @@ export async function runOfflineEvaluation(
}));
const storeResult = await storeEvaluation(
spanId,
- data.result || [],
+ evaluations,
metaBase,
databaseConfigId,
customTypes
@@ -1000,7 +1060,7 @@ export async function runOfflineEvaluation(
return {
success: true,
- evaluations: data.result || [],
+ evaluations,
contextApplied: {
ruleMatched: matchingRuleIds.length > 0,
matchingRuleIds,
diff --git a/src/client/src/lib/platform/evaluation/offline-auth.ts b/src/client/src/lib/platform/evaluation/offline-auth.ts
new file mode 100644
index 00000000..5409eb80
--- /dev/null
+++ b/src/client/src/lib/platform/evaluation/offline-auth.ts
@@ -0,0 +1,56 @@
+import { getAPIKeyInfo } from "@/lib/platform/api-keys";
+import { getEvaluationConfigByDbConfigId } from "./config";
+import getMessage from "@/constants/messages";
+import asaw from "@/utils/asaw";
+import { errorResponse } from "@/helpers/server/response";
+import { EvaluationConfigWithSecret } from "@/types/evaluation";
+
+export type OfflineEvaluationTypeConfig = EvaluationConfigWithSecret & {
+ evaluationTypes?: any[];
+};
+
+/**
+ * Authenticates an offline/SDK request via its Bearer API key. Shared by
+ * every /api/evaluation/offline/types handler so the auth check and its
+ * 401 responses can't drift between them.
+ */
+export async function authenticateOfflineApiKey(
+ request: Request
+): Promise<{ error: Response } | { databaseConfigId: string }> {
+ const authorizationHeader = request.headers.get("Authorization") || "";
+ if (!authorizationHeader.startsWith("Bearer ")) {
+ return { error: errorResponse(getMessage().NO_API_KEY, 401) };
+ }
+ const apiKey = authorizationHeader.replace(/^Bearer /, "").trim();
+ if (!apiKey) {
+ return { error: errorResponse(getMessage().NO_API_KEY, 401) };
+ }
+ const [keyErr, apiInfo] = await getAPIKeyInfo({ apiKey });
+ if (keyErr || !apiInfo?.databaseConfigId) {
+ return { error: errorResponse(getMessage().NO_API_KEY, 401) };
+ }
+ return { databaseConfigId: apiInfo.databaseConfigId };
+}
+
+/**
+ * Loads the evaluation config for an authenticated offline/SDK request's
+ * own database config, scoped by databaseConfigId. `notConfiguredResponse`
+ * lets each caller shape its own "not configured" response (e.g. GET
+ * degrades gracefully with a 200, while mutating routes return a 400) while
+ * still sharing the lookup and error-detection logic.
+ */
+export async function loadOfflineEvaluationConfig(
+ databaseConfigId: string,
+ notConfiguredResponse: () => Response
+): Promise<{ error: Response } | { config: OfflineEvaluationTypeConfig }> {
+ const [configErr, config] = await asaw(
+ getEvaluationConfigByDbConfigId(databaseConfigId, true)
+ );
+ if (configErr || !config?.id) {
+ return { error: notConfiguredResponse() };
+ }
+ return { config: config as OfflineEvaluationTypeConfig };
+}
+
+export const EVALUATION_NOT_CONFIGURED_MESSAGE =
+ "Evaluation not configured. Set up evaluation in the OpenLIT dashboard first.";
diff --git a/src/client/src/lib/platform/evaluation/rule-engine-context.ts b/src/client/src/lib/platform/evaluation/rule-engine-context.ts
index 57162e70..6afa903a 100644
--- a/src/client/src/lib/platform/evaluation/rule-engine-context.ts
+++ b/src/client/src/lib/platform/evaluation/rule-engine-context.ts
@@ -65,17 +65,20 @@ export interface RuleEngineContextResult {
}
/**
- * Fetches context content from the rule engine for a given trace.
- * Evaluates rules using trace attributes and returns concatenated context
- * content from matching context entities.
+ * Fetches context content from the rule engine for a set of rule-engine
+ * input fields. Evaluates rules against those fields and returns
+ * concatenated context content from matching context entities.
+ *
+ * This is the shared primitive behind both the trace-based (real-time/auto)
+ * and fields-based (offline/SDK) evaluation paths, so both resolve context
+ * identically given equivalent input.
*
* @returns Context contents and matching rule IDs (for display and storage)
*/
-export async function getContextFromRuleEngineForTrace(
- trace: TraceRow,
+export async function getContextFromFields(
+ fields: Record<string, string | number | boolean>,
databaseConfigId?: string
): Promise<RuleEngineContextResult> {
- const fields = extractRuleEngineFieldsFromTrace(trace);
if (Object.keys(fields).length === 0) {
return { contextContents: [], matchingRuleIds: [], contextEntityIds: [] };
}
@@ -117,21 +120,39 @@ export async function getContextFromRuleEngineForTrace(
}
}
+/**
+ * Fetches context content from the rule engine for a given trace.
+ * Evaluates rules using trace attributes and returns concatenated context
+ * content from matching context entities.
+ *
+ * @returns Context contents and matching rule IDs (for display and storage)
+ */
+export async function getContextFromRuleEngineForTrace(
+ trace: TraceRow,
+ databaseConfigId?: string
+): Promise<RuleEngineContextResult> {
+ return getContextFromFields(
+ extractRuleEngineFieldsFromTrace(trace),
+ databaseConfigId
+ );
+}
+
export interface RuleWithPriority {
ruleId: string;
priority: number;
}
/**
- * Fetches context from rules in priority order. Used when evaluation types
- * config specifies which rules to use for auto evaluation.
+ * Fetches context from rules in priority order, given rule-engine input
+ * fields directly. Shared primitive behind both the trace-based
+ * (real-time/auto) and fields-based (offline/SDK) evaluation paths.
*/
-export async function getContextFromRulesWithPriority(
- trace: TraceRow,
+export async function getContextFromRulesWithPriorityForFields(
+ fields: Record<string, string | number | boolean>,
rulesWithPriority: RuleWithPriority[],
databaseConfigId?: string
): Promise<RuleEngineContextResult> {
- const result = await getContextFromRuleEngineForTrace(trace, databaseConfigId);
+ const result = await getContextFromFields(fields, databaseConfigId);
if (rulesWithPriority.length === 0) return result;
const priorityByRule = new Map(
@@ -148,7 +169,6 @@ export async function getContextFromRulesWithPriority(
);
// Re-fetch with entity data to get ordered context
- const fields = extractRuleEngineFieldsFromTrace(trace);
if (Object.keys(fields).length === 0) {
return {
contextContents: [],
@@ -194,3 +214,19 @@ export async function getContextFromRulesWithPriority(
return result;
}
}
+
+/**
+ * Fetches context from rules in priority order. Used when evaluation types
+ * config specifies which rules to use for auto evaluation.
+ */
+export async function getContextFromRulesWithPriority(
+ trace: TraceRow,
+ rulesWithPriority: RuleWithPriority[],
+ databaseConfigId?: string
+): Promise<RuleEngineContextResult> {
+ return getContextFromRulesWithPriorityForFields(
+ extractRuleEngineFieldsFromTrace(trace),
+ rulesWithPriority,
+ databaseConfigId
+ );
+}
diff --git a/src/client/src/lib/platform/evaluation/threshold.ts b/src/client/src/lib/platform/evaluation/threshold.ts
new file mode 100644
index 00000000..e8f6f69c
--- /dev/null
+++ b/src/client/src/lib/platform/evaluation/threshold.ts
@@ -0,0 +1,21 @@
+/**
+ * Validates and clamps a per-type threshold override. Returns undefined when
+ * no value was provided (caller should fall back to the request/global
+ * default), or NaN when a value was provided but isn't a usable number
+ * (caller should reject the write).
+ */
+export function normalizeThresholdScore(value: unknown): number | undefined {
+ if (value === undefined || value === null || value === "") {
+ return undefined;
+ }
+
+ const numeric =
+ typeof value === "number" ? value : Number.parseFloat(String(value));
+
+ if (!Number.isFinite(numeric)) {
+ return Number.NaN;
+ }
+ if (numeric < 0) return 0;
+ if (numeric > 1) return 1;
+ return numeric;
+}
diff --git a/src/client/src/lib/platform/evaluation/type-config.ts b/src/client/src/lib/platform/evaluation/type-config.ts
new file mode 100644
index 00000000..b4ac27f3
--- /dev/null
+++ b/src/client/src/lib/platform/evaluation/type-config.ts
@@ -0,0 +1,123 @@
+import { jsonParse, jsonStringify } from "@/utils/json";
+import { syncRuleEntitiesFromConfig } from "./sync-rule-entities";
+
+export interface RuleWithPriority {
+ ruleId: string;
+ priority: number;
+}
+
+export interface EvaluationTypeConfig {
+ id: string;
+ enabled: boolean;
+ isCustom?: boolean;
+ label?: string;
+ description?: string;
+ rules?: RuleWithPriority[];
+ ruleId?: string;
+ priority?: number;
+ defaultPrompt?: string;
+ prompt?: string;
+ thresholdScore?: number;
+}
+
+/**
+ * Normalizes a raw rules array (as received from a request body) into
+ * validated { ruleId, priority } pairs, dropping entries without a ruleId.
+ */
+export function normalizeRules(rules: any[]): RuleWithPriority[] {
+ if (!Array.isArray(rules)) return [];
+ return rules
+ .filter((r: any) => r?.ruleId)
+ .map((r: any) => ({
+ ruleId: r.ruleId,
+ priority: Number(r.priority) || 0,
+ }));
+}
+
+/**
+ * Normalizes a single raw evaluation type entry (as received from a request
+ * body) into a persistable EvaluationTypeConfig. The threshold is validated
+ * and normalized by the caller (via normalizeThresholdScore) beforehand,
+ * since only the caller knows how to surface a validation error.
+ */
+export function normalizeTypeConfig(
+ t: any,
+ thresholdScore: number | undefined
+): EvaluationTypeConfig {
+ const rules = t.rules?.length
+ ? normalizeRules(t.rules)
+ : t.ruleId
+ ? [{ ruleId: t.ruleId, priority: Number(t.priority) || 0 }]
+ : [];
+ const config: EvaluationTypeConfig = {
+ id: t.id,
+ enabled: !!t.enabled,
+ rules,
+ };
+ if (thresholdScore !== undefined) config.thresholdScore = thresholdScore;
+ // Preserve custom type metadata
+ if (t.isCustom) {
+ config.isCustom = true;
+ if (t.label) config.label = String(t.label).trim();
+ if (t.description) config.description = String(t.description).trim();
+ if (t.prompt) config.prompt = String(t.prompt).trim();
+ } else if (t.prompt) {
+ config.prompt = String(t.prompt).trim();
+ }
+ return config;
+}
+
+/**
+ * Upserts a single type into an existing evaluationTypes list by id,
+ * appending it when no entry with that id exists yet.
+ */
+export function mergeTypeIntoList(
+ types: EvaluationTypeConfig[],
+ updated: EvaluationTypeConfig
+): EvaluationTypeConfig[] {
+ const idx = types.findIndex((t) => t.id === updated.id);
+ if (idx >= 0) {
+ const next = [...types];
+ next[idx] = updated;
+ return next;
+ }
+ return [...types, updated];
+}
+
+/**
+ * Upserts a batch of types into an existing list by id. Used by the
+ * API-key offline create endpoint so posting a partial list cannot wipe
+ * unrelated types (and their thresholdScore / rules) already in meta.
+ */
+export function upsertEvaluationTypes(
+ existing: EvaluationTypeConfig[],
+ incoming: EvaluationTypeConfig[]
+): EvaluationTypeConfig[] {
+ return incoming.reduce(
+ (acc, type) => mergeTypeIntoList(acc, type),
+ existing
+ );
+}
+
+/**
+ * Persists an evaluationTypes array into an evaluation config's meta JSON
+ * and re-syncs rule-engine entities. Shared by every route (dashboard and
+ * API-key-authed) that creates or updates evaluation types, so the
+ * persistence step behaves identically regardless of caller.
+ */
+export async function persistEvaluationTypes(
+ configId: string,
+ configMeta: string | null | undefined,
+ evaluationTypes: EvaluationTypeConfig[]
+): Promise<void> {
+ const prisma = (await import("@/lib/prisma")).default;
+ const meta = jsonParse(configMeta || "{}") as Record<string, any>;
+ meta.evaluationTypes = evaluationTypes;
+
+ await prisma.evaluationConfigs.update({
+ where: { id: configId },
+ data: { meta: jsonStringify(meta) },
+ });
+
+ await syncRuleEntitiesFromConfig();
+}