Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
Feat: use platform-configured evaluation types for offline evaluations (#1382)
Linda Oraegbunam11f177d71fa9
diff --git a/sdk/python/src/openlit/evals/offline.py b/sdk/python/src/openlit/evals/offline.pyindex b0422fc4..42aca428 100644--- a/sdk/python/src/openlit/evals/offline.py+++ b/sdk/python/src/openlit/evals/offline.py@@ -350,6 +350,7 @@ def fetch_eval_types(description=t.get("description", ""),enabled=t.get("enabled", False),is_custom=t.get("is_custom", False),+ threshold_score=t.get("threshold_score"),)for t in data.get("eval_types", [])]diff --git a/sdk/python/src/openlit/evals/offline_types.py b/sdk/python/src/openlit/evals/offline_types.pyindex 936914ee..efa0f582 100644--- a/sdk/python/src/openlit/evals/offline_types.py+++ b/sdk/python/src/openlit/evals/offline_types.py@@ -113,6 +113,7 @@ class EvalType(BaseModel):description: str = ""enabled: bool = Falseis_custom: bool = False+ threshold_score: Optional[float] = Noneclass BatchEvalResult(BaseModel):diff --git a/sdk/python/tests/test_offline_eval.py b/sdk/python/tests/test_offline_eval.pyindex 6dbd3242..2f6a846d 100644--- a/sdk/python/tests/test_offline_eval.py+++ b/sdk/python/tests/test_offline_eval.py@@ -309,6 +309,7 @@ def test_fetch_eval_types(mock_get):"description": "Detects hallucinated content","enabled": True,"is_custom": False,+ "threshold_score": 0.7,},{"id": "custom_safety",@@ -330,8 +331,10 @@ def test_fetch_eval_types(mock_get):assert len(types) == 2assert types[0].id == "hallucination"assert types[0].is_custom is False+ assert types[0].threshold_score == 0.7assert types[1].id == "custom_safety"assert types[1].is_custom is True+ assert types[1].threshold_score is None# --- Terminal output ---diff --git a/sdk/typescript/src/evals/__tests__/offline.test.ts b/sdk/typescript/src/evals/__tests__/offline.test.tsindex 917010eb..c9a58980 100644--- a/sdk/typescript/src/evals/__tests__/offline.test.ts+++ b/sdk/typescript/src/evals/__tests__/offline.test.ts@@ -346,7 +346,14 @@ describe('fetchEvalTypes', () => {test('returns parsed eval types', async () => {global.fetch = mockFetchResponse({eval_types: [- { id: 'hallucination', label: 'Hallucination', description: 'Detect hallucinated facts', enabled: true, is_custom: false },+ {+ id: 'hallucination',+ label: 'Hallucination',+ description: 'Detect hallucinated facts',+ enabled: true,+ is_custom: false,+ threshold_score: 0.7,+ },{ id: 'custom_1', label: 'Custom', description: 'User-defined', enabled: true, is_custom: true },],});@@ -355,7 +362,9 @@ describe('fetchEvalTypes', () => {expect(types).toHaveLength(2);expect(types[0].id).toBe('hallucination');expect(types[0].isCustom).toBe(false);+ expect(types[0].thresholdScore).toBe(0.7);expect(types[1].isCustom).toBe(true);+ expect(types[1].thresholdScore).toBeUndefined();});test('throws on 401', async () => {diff --git a/sdk/typescript/src/evals/offline.ts b/sdk/typescript/src/evals/offline.tsindex 75873895..5298c1a7 100644--- a/sdk/typescript/src/evals/offline.ts+++ b/sdk/typescript/src/evals/offline.ts@@ -353,6 +353,9 @@ export async function fetchEvalTypes(options: EvalTypesOptions = {}): Promise<Evdescription: t.description || '',enabled: !!t.enabled,isCustom: !!t.is_custom,+ ...(typeof t.threshold_score === 'number'+ ? { thresholdScore: t.threshold_score }+ : {}),}));} catch (e: any) {if (e?.name === 'AbortError') {diff --git a/sdk/typescript/src/evals/types.ts b/sdk/typescript/src/evals/types.tsindex 8a9facf8..3e55947b 100644--- a/sdk/typescript/src/evals/types.ts+++ b/sdk/typescript/src/evals/types.ts@@ -27,6 +27,7 @@ export interface EvalType {description: string;enabled: boolean;isCustom: boolean;+ thresholdScore?: number;}export interface BatchEvalResult {diff --git a/src/client/src/__tests__/app/api/evaluation/offline/types/[id]/route.test.ts b/src/client/src/__tests__/app/api/evaluation/offline/types/[id]/route.test.tsnew file mode 100644index 00000000..96e09e14--- /dev/null+++ b/src/client/src/__tests__/app/api/evaluation/offline/types/[id]/route.test.ts@@ -0,0 +1,238 @@+jest.mock("@/lib/platform/api-keys", () => ({+ getAPIKeyInfo: jest.fn(),+}));+jest.mock("@/lib/platform/evaluation/config", () => ({+ getEvaluationConfigByDbConfigId: jest.fn(),+}));+jest.mock("@/lib/platform/evaluation/sync-rule-entities", () => ({+ syncRuleEntitiesFromConfig: jest.fn().mockResolvedValue(undefined),+}));+jest.mock("@/lib/posthog", () => ({+ __esModule: true,+ default: {+ fireEvent: jest.fn(),+ },+}));+jest.mock("@/constants/messages", () => ({+ __esModule: true,+ default: jest.fn(() => ({+ NO_API_KEY: "No API key provided",+ EVALUATION_THRESHOLD_SCORE_INVALID:+ "Threshold score must be a number between 0 and 1.",+ })),+}));+jest.mock("@/utils/json", () => ({+ jsonParse: jest.fn((v: string) => {+ try {+ return JSON.parse(v);+ } catch {+ return {};+ }+ }),+ jsonStringify: jest.fn((v: unknown) => JSON.stringify(v)),+}));+jest.mock("@/utils/asaw", () =>+ jest.fn(async (promise: Promise<any>) => {+ try {+ const result = await promise;+ return [null, result];+ } catch (err) {+ return [err, null];+ }+ })+);+jest.mock("@/lib/prisma", () => ({+ __esModule: true,+ default: {+ evaluationConfigs: {+ update: jest.fn().mockResolvedValue({}),+ },+ },+}));++import { PATCH, OPTIONS } from "@/app/api/evaluation/offline/types/[id]/route";+import { getAPIKeyInfo } from "@/lib/platform/api-keys";+import { getEvaluationConfigByDbConfigId } from "@/lib/platform/evaluation/config";+import prisma from "@/lib/prisma";++class TestHeaders {+ private values = new Map<string, string>();++ constructor(headers?: Record<string, string>) {+ Object.entries(headers || {}).forEach(([key, value]) => {+ this.values.set(key.toLowerCase(), value);+ });+ }++ get(key: string) {+ return this.values.get(key.toLowerCase()) ?? null;+ }+}++class TestResponse {+ status: number;+ headers: TestHeaders;+ private body: unknown;++ constructor(+ body?: unknown,+ init?: { status?: number; headers?: Record<string, string> }+ ) {+ this.body = body;+ this.status = init?.status ?? 200;+ this.headers = new TestHeaders(init?.headers);+ }++ static json(+ body: unknown,+ init?: { status?: number; headers?: Record<string, string> }+ ) {+ return new TestResponse(body, init);+ }++ async json() {+ return this.body;+ }+}++(global as any).Response = TestResponse;++function makeRequest({+ authorization = "Bearer openlit-test-key",+ body = {},+}: {+ authorization?: string;+ body?: Record<string, unknown>;+} = {}) {+ return {+ headers: {+ get: (key: string) => {+ if (key.toLowerCase() === "authorization") return authorization;+ return null;+ },+ },+ json: jest.fn().mockResolvedValue(body),+ } as unknown as Request;+}++function makeInvalidJsonRequest(authorization = "Bearer openlit-test-key") {+ return {+ headers: {+ get: (key: string) => (key.toLowerCase() === "authorization" ? authorization : null),+ },+ json: jest.fn().mockRejectedValue(new SyntaxError("Unexpected token")),+ } as unknown as Request;+}++describe("PATCH /api/evaluation/offline/types/[id]", () => {+ beforeEach(() => {+ jest.clearAllMocks();+ });++ it("returns 401 when no Bearer token is provided", async () => {+ const response = await PATCH(makeRequest({ authorization: "" }), {+ params: { id: "toxicity" },+ });+ expect(response.status).toBe(401);+ });++ it("returns 401 when API key is invalid", async () => {+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([new Error("Not found"), null]);+ const response = await PATCH(makeRequest({ body: { enabled: true } }), {+ params: { id: "toxicity" },+ });+ expect(response.status).toBe(401);+ });++ it("returns 400 on invalid JSON body", async () => {+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([null, { databaseConfigId: "db-1" }]);+ const response = await PATCH(makeInvalidJsonRequest(), {+ params: { id: "toxicity" },+ });+ expect(response.status).toBe(400);+ const data = await response.json();+ expect(data.err).toBe("Invalid JSON body");+ });++ it("rejects a non-numeric thresholdScore without calling getEvaluationConfigByDbConfigId", async () => {+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([null, { databaseConfigId: "db-1" }]);+ const response = await PATCH(+ makeRequest({ body: { thresholdScore: "bad" } }),+ { params: { id: "toxicity" } }+ );+ expect(response.status).toBe(400);+ const data = await response.json();+ expect(data.err).toBe("Threshold score must be a number between 0 and 1.");+ expect(getEvaluationConfigByDbConfigId).not.toHaveBeenCalled();+ });++ it("returns 400 when evaluation config is not found", async () => {+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([null, { databaseConfigId: "db-1" }]);+ (getEvaluationConfigByDbConfigId as jest.Mock).mockRejectedValue(new Error("not found"));++ const response = await PATCH(makeRequest({ body: { enabled: true } }), {+ params: { id: "toxicity" },+ });+ expect(response.status).toBe(400);+ });++ it("scopes the update to the API key's own database config and clamps thresholdScore", async () => {+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([null, { databaseConfigId: "db-1" }]);+ (getEvaluationConfigByDbConfigId as jest.Mock).mockResolvedValue({+ id: "cfg-1",+ meta: JSON.stringify({+ evaluationTypes: [{ id: "toxicity", enabled: true, rules: [] }],+ }),+ });++ const response = await PATCH(+ makeRequest({ body: { thresholdScore: 1.5 } }),+ { params: { id: "toxicity" } }+ );++ expect(response.status).toBe(200);+ const data = await response.json();+ expect(data.data.thresholdScore).toBe(1);+ expect(getEvaluationConfigByDbConfigId).toHaveBeenCalledWith("db-1", true);++ const updateCall = (prisma.evaluationConfigs.update as jest.Mock).mock.calls[0][0];+ expect(updateCall.where).toEqual({ id: "cfg-1" });+ const persistedMeta = JSON.parse(updateCall.data.meta);+ expect(persistedMeta.evaluationTypes[0]).toMatchObject({+ id: "toxicity",+ thresholdScore: 1,+ });+ });++ it("creates a new type entry when the id doesn't already exist", async () => {+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([null, { databaseConfigId: "db-1" }]);+ (getEvaluationConfigByDbConfigId as jest.Mock).mockResolvedValue({+ id: "cfg-1",+ meta: JSON.stringify({ evaluationTypes: [] }),+ });++ const response = await PATCH(+ makeRequest({+ body: { isCustom: true, label: "Custom Check", enabled: true },+ }),+ { params: { id: "custom_check" } }+ );++ expect(response.status).toBe(200);+ const data = await response.json();+ expect(data.data).toMatchObject({+ id: "custom_check",+ isCustom: true,+ label: "Custom Check",+ enabled: true,+ });+ });+});++describe("OPTIONS /api/evaluation/offline/types/[id]", () => {+ it("allows PATCH and OPTIONS", async () => {+ const response = await OPTIONS();+ expect(response.status).toBe(200);+ expect(response.headers.get("Access-Control-Allow-Methods")).toContain("PATCH");+ });+});diff --git a/src/client/src/__tests__/app/api/evaluation/offline/types/route.test.ts b/src/client/src/__tests__/app/api/evaluation/offline/types/route.test.tsnew file mode 100644index 00000000..46fd3e84--- /dev/null+++ b/src/client/src/__tests__/app/api/evaluation/offline/types/route.test.ts@@ -0,0 +1,303 @@+jest.mock("@/lib/platform/api-keys", () => ({+ getAPIKeyInfo: jest.fn(),+}));+jest.mock("@/lib/platform/evaluation/config", () => ({+ getEvaluationConfigByDbConfigId: jest.fn(),+}));+jest.mock("@/lib/platform/evaluation/sync-rule-entities", () => ({+ syncRuleEntitiesFromConfig: jest.fn().mockResolvedValue(undefined),+}));+jest.mock("@/lib/posthog", () => ({+ __esModule: true,+ default: {+ fireEvent: jest.fn(),+ },+}));+jest.mock("@/constants/messages", () => ({+ __esModule: true,+ default: jest.fn(() => ({+ NO_API_KEY: "No API key provided",+ EVALUATION_THRESHOLD_SCORE_INVALID:+ "Threshold score must be a number between 0 and 1.",+ })),+}));+jest.mock("@/utils/json", () => ({+ jsonParse: jest.fn((v: string) => {+ try {+ return JSON.parse(v);+ } catch {+ return {};+ }+ }),+ jsonStringify: jest.fn((v: unknown) => JSON.stringify(v)),+}));+jest.mock("@/utils/asaw", () =>+ jest.fn(async (promise: Promise<any>) => {+ try {+ const result = await promise;+ return [null, result];+ } catch (err) {+ return [err, null];+ }+ })+);+jest.mock("@/lib/prisma", () => ({+ __esModule: true,+ default: {+ evaluationConfigs: {+ update: jest.fn().mockResolvedValue({}),+ },+ },+}));++import { GET, POST, OPTIONS } from "@/app/api/evaluation/offline/types/route";+import { getAPIKeyInfo } from "@/lib/platform/api-keys";+import { getEvaluationConfigByDbConfigId } from "@/lib/platform/evaluation/config";+import prisma from "@/lib/prisma";++class TestHeaders {+ private values = new Map<string, string>();++ constructor(headers?: Record<string, string>) {+ Object.entries(headers || {}).forEach(([key, value]) => {+ this.values.set(key.toLowerCase(), value);+ });+ }++ get(key: string) {+ return this.values.get(key.toLowerCase()) ?? null;+ }+}++class TestResponse {+ status: number;+ headers: TestHeaders;+ private body: unknown;++ constructor(+ body?: unknown,+ init?: { status?: number; headers?: Record<string, string> }+ ) {+ this.body = body;+ this.status = init?.status ?? 200;+ this.headers = new TestHeaders(init?.headers);+ }++ static json(+ body: unknown,+ init?: { status?: number; headers?: Record<string, string> }+ ) {+ return new TestResponse(body, init);+ }++ async json() {+ return this.body;+ }+}++(global as any).Response = TestResponse;++function makeRequest({+ authorization = "Bearer openlit-test-key",+ body = {},+}: {+ authorization?: string;+ body?: Record<string, unknown>;+} = {}) {+ return {+ headers: {+ get: (key: string) => {+ if (key.toLowerCase() === "authorization") return authorization;+ return null;+ },+ },+ json: jest.fn().mockResolvedValue(body),+ } as unknown as Request;+}++function makeInvalidJsonRequest(authorization = "Bearer openlit-test-key") {+ return {+ headers: {+ get: (key: string) => (key.toLowerCase() === "authorization" ? authorization : null),+ },+ json: jest.fn().mockRejectedValue(new SyntaxError("Unexpected token")),+ } as unknown as Request;+}++describe("GET /api/evaluation/offline/types", () => {+ beforeEach(() => {+ jest.clearAllMocks();+ });++ it("returns 401 when no Bearer token is provided", async () => {+ const response = await GET(makeRequest({ authorization: "" }));+ expect(response.status).toBe(401);+ });++ it("returns 401 when API key is invalid", async () => {+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([new Error("Not found"), null]);+ const response = await GET(makeRequest());+ expect(response.status).toBe(401);+ });++ it("returns configured:false when evaluation config is not found", async () => {+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([null, { databaseConfigId: "db-1" }]);+ (getEvaluationConfigByDbConfigId as jest.Mock).mockRejectedValue(new Error("not found"));++ const response = await GET(makeRequest());+ expect(response.status).toBe(200);+ const data = await response.json();+ expect(data.configured).toBe(false);+ expect(data.eval_types).toEqual([]);+ });++ it("returns the configured evaluation types", async () => {+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([null, { databaseConfigId: "db-1" }]);+ (getEvaluationConfigByDbConfigId as jest.Mock).mockResolvedValue({+ id: "cfg-1",+ evaluationTypes: [+ {+ id: "hallucination",+ label: "Hallucination",+ enabled: true,+ isCustom: false,+ thresholdScore: 0.7,+ },+ {+ id: "toxicity",+ label: "Toxicity",+ enabled: true,+ isCustom: false,+ },+ ],+ });++ const response = await GET(makeRequest());+ expect(response.status).toBe(200);+ const data = await response.json();+ expect(data.eval_types).toEqual([+ {+ id: "hallucination",+ label: "Hallucination",+ description: "",+ enabled: true,+ is_custom: false,+ threshold_score: 0.7,+ },+ {+ id: "toxicity",+ label: "Toxicity",+ description: "",+ enabled: true,+ is_custom: false,+ },+ ]);+ });+});++describe("POST /api/evaluation/offline/types", () => {+ beforeEach(() => {+ jest.clearAllMocks();+ });++ it("returns 401 when no Bearer token is provided", async () => {+ const response = await POST(makeRequest({ authorization: "" }));+ expect(response.status).toBe(401);+ });++ it("returns 401 when API key is invalid", async () => {+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([new Error("Not found"), null]);+ const response = await POST(makeRequest({ body: { types: [] } }));+ expect(response.status).toBe(401);+ });++ it("returns 400 on invalid JSON body", async () => {+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([null, { databaseConfigId: "db-1" }]);+ const response = await POST(makeInvalidJsonRequest());+ expect(response.status).toBe(400);+ const data = await response.json();+ expect(data.err).toBe("Invalid JSON body");+ });++ it("returns 400 when types is not an array", async () => {+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([null, { databaseConfigId: "db-1" }]);+ const response = await POST(makeRequest({ body: { types: "not-an-array" } }));+ expect(response.status).toBe(400);+ const data = await response.json();+ expect(data.err).toBe("Invalid types array");+ });++ it("rejects a non-numeric thresholdScore without calling getEvaluationConfigByDbConfigId", async () => {+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([null, { databaseConfigId: "db-1" }]);+ const response = await POST(+ makeRequest({+ body: { types: [{ id: "toxicity", enabled: true, thresholdScore: "bad" }] },+ })+ );+ expect(response.status).toBe(400);+ const data = await response.json();+ expect(data.err).toBe("Threshold score must be a number between 0 and 1.");+ expect(getEvaluationConfigByDbConfigId).not.toHaveBeenCalled();+ });++ it("returns 400 when evaluation config is not found", async () => {+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([null, { databaseConfigId: "db-1" }]);+ (getEvaluationConfigByDbConfigId as jest.Mock).mockRejectedValue(new Error("not found"));++ const response = await POST(+ makeRequest({ body: { types: [{ id: "hallucination", enabled: true }] } })+ );+ expect(response.status).toBe(400);+ const data = await response.json();+ expect(data.err).toBe(+ "Evaluation not configured. Set up evaluation in the OpenLIT dashboard first."+ );+ });++ it("normalizes and upserts types scoped to the API key's own database config", async () => {+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([null, { databaseConfigId: "db-1" }]);+ (getEvaluationConfigByDbConfigId as jest.Mock).mockResolvedValue({+ id: "cfg-1",+ meta: JSON.stringify({+ evaluationTypes: [+ { id: "bias", enabled: true, rules: [], thresholdScore: 0.4 },+ ],+ }),+ });++ const response = await POST(+ makeRequest({+ body: {+ types: [{ id: "toxicity", enabled: true, thresholdScore: 1.5 }],+ },+ })+ );++ expect(response.status).toBe(200);+ const data = await response.json();+ expect(data.data).toEqual([+ expect.objectContaining({ id: "toxicity", enabled: true, thresholdScore: 1 }),+ ]);+ expect(getEvaluationConfigByDbConfigId).toHaveBeenCalledWith("db-1", true);+ const updateCall = (prisma.evaluationConfigs.update as jest.Mock).mock.calls[0][0];+ expect(updateCall.where).toEqual({ id: "cfg-1" });+ const persistedMeta = JSON.parse(updateCall.data.meta);+ // Partial POST upserts without wiping unrelated types/thresholds.+ expect(persistedMeta.evaluationTypes).toEqual(+ expect.arrayContaining([+ expect.objectContaining({ id: "bias", thresholdScore: 0.4 }),+ expect.objectContaining({ id: "toxicity", thresholdScore: 1 }),+ ])+ );+ expect(persistedMeta.evaluationTypes).toHaveLength(2);+ });+});++describe("OPTIONS /api/evaluation/offline/types", () => {+ it("allows GET, POST, and OPTIONS", async () => {+ const response = await OPTIONS();+ expect(response.status).toBe(200);+ expect(response.headers.get("Access-Control-Allow-Methods")).toContain("POST");+ expect(response.headers.get("Access-Control-Allow-Methods")).toContain("GET");+ });+});diff --git a/src/client/src/__tests__/app/api/evaluation/types/[id]/route.test.ts b/src/client/src/__tests__/app/api/evaluation/types/[id]/route.test.tsnew file mode 100644index 00000000..edc0f163--- /dev/null+++ b/src/client/src/__tests__/app/api/evaluation/types/[id]/route.test.ts@@ -0,0 +1,184 @@+jest.mock("@/lib/platform/evaluation/config", () => ({+ getEvaluationConfig: jest.fn(),+}));+jest.mock("@/lib/platform/evaluation/sync-rule-entities", () => ({+ syncRuleEntitiesFromConfig: jest.fn().mockResolvedValue(undefined),+}));+jest.mock("@/lib/posthog", () => ({+ __esModule: true,+ default: {+ fireEvent: jest.fn(),+ },+}));+jest.mock("@/constants/messages", () => ({+ __esModule: true,+ default: jest.fn(() => ({+ EVALUATION_THRESHOLD_SCORE_INVALID:+ "Threshold score must be a number between 0 and 1.",+ })),+}));+jest.mock("@/utils/json", () => ({+ jsonParse: jest.fn((v: string) => {+ try {+ return JSON.parse(v);+ } catch {+ return {};+ }+ }),+ jsonStringify: jest.fn((v: unknown) => JSON.stringify(v)),+}));+jest.mock("@/utils/asaw", () =>+ jest.fn(async (promise: Promise<any>) => {+ try {+ const result = await promise;+ return [null, result];+ } catch (err) {+ return [err, null];+ }+ })+);+jest.mock("@/lib/prisma", () => ({+ __esModule: true,+ default: {+ evaluationConfigs: {+ update: jest.fn().mockResolvedValue({}),+ },+ },+}));++import { PATCH } from "@/app/api/evaluation/types/[id]/route";+import { getEvaluationConfig } from "@/lib/platform/evaluation/config";+import prisma from "@/lib/prisma";++class TestResponse {+ status: number;+ private body: unknown;++ constructor(body?: unknown, init?: { status?: number }) {+ this.body = body;+ this.status = init?.status ?? 200;+ }++ static json(body: unknown, init?: { status?: number }) {+ return new TestResponse(body, init);+ }++ async json() {+ return this.body;+ }+}++(global as any).Response = TestResponse;++function makeRequest(body: Record<string, unknown>) {+ return {+ json: jest.fn().mockResolvedValue(body),+ } as unknown as Request;+}++function makeInvalidJsonRequest() {+ return {+ json: jest.fn().mockRejectedValue(new SyntaxError("Unexpected token")),+ } as unknown as Request;+}++describe("PATCH /api/evaluation/types/[id]", () => {+ beforeEach(() => {+ jest.clearAllMocks();+ });++ it("returns 400 on invalid JSON body", async () => {+ const response = await PATCH(makeInvalidJsonRequest() as any, {+ params: { id: "toxicity" },+ });+ expect(response.status).toBe(400);+ const data = await response.json();+ expect(data.err).toBe("Invalid JSON body");+ });++ it("rejects a non-numeric thresholdScore without calling getEvaluationConfig", async () => {+ const response = await PATCH(+ makeRequest({ thresholdScore: "not-a-number" }) as any,+ { params: { id: "toxicity" } }+ );++ expect(response.status).toBe(400);+ const data = await response.json();+ expect(data.err).toBe("Threshold score must be a number between 0 and 1.");+ expect(getEvaluationConfig).not.toHaveBeenCalled();+ });++ it("returns 400 when evaluation config is not found", async () => {+ (getEvaluationConfig as jest.Mock).mockRejectedValue(new Error("not found"));++ const response = await PATCH(makeRequest({ enabled: true }) as any, {+ params: { id: "toxicity" },+ });++ expect(response.status).toBe(400);+ const data = await response.json();+ expect(data.err).toBe("Evaluation config not found");+ });++ it("clamps an out-of-range thresholdScore and persists it for an existing type", async () => {+ (getEvaluationConfig as jest.Mock).mockResolvedValue({+ id: "cfg-1",+ meta: JSON.stringify({+ evaluationTypes: [{ id: "toxicity", enabled: true, rules: [] }],+ }),+ });++ const response = await PATCH(+ makeRequest({ thresholdScore: 1.5 }) as any,+ { params: { id: "toxicity" } }+ );++ expect(response.status).toBe(200);+ const data = await response.json();+ expect(data.data.thresholdScore).toBe(1);++ const updateCall = (prisma.evaluationConfigs.update as jest.Mock).mock.calls[0][0];+ const persistedMeta = JSON.parse(updateCall.data.meta);+ expect(persistedMeta.evaluationTypes[0]).toMatchObject({+ id: "toxicity",+ thresholdScore: 1,+ });+ });++ it("preserves the existing thresholdScore when the body omits it", async () => {+ (getEvaluationConfig as jest.Mock).mockResolvedValue({+ id: "cfg-1",+ meta: JSON.stringify({+ evaluationTypes: [+ { id: "toxicity", enabled: true, rules: [], thresholdScore: 0.8 },+ ],+ }),+ });++ const response = await PATCH(makeRequest({ enabled: false }) as any, {+ params: { id: "toxicity" },+ });++ const data = await response.json();+ expect(data.data.thresholdScore).toBe(0.8);+ });++ it("clears the existing thresholdScore when the body sends null", async () => {+ (getEvaluationConfig as jest.Mock).mockResolvedValue({+ id: "cfg-1",+ meta: JSON.stringify({+ evaluationTypes: [+ { id: "toxicity", enabled: true, rules: [], thresholdScore: 0.8 },+ ],+ }),+ });++ const response = await PATCH(+ makeRequest({ thresholdScore: null }) as any,+ { params: { id: "toxicity" } }+ );++ const data = await response.json();+ expect(data.data.thresholdScore).toBeUndefined();+ });+});diff --git a/src/client/src/__tests__/app/api/evaluation/types/route.test.ts b/src/client/src/__tests__/app/api/evaluation/types/route.test.tsnew file mode 100644index 00000000..993d6e72--- /dev/null+++ b/src/client/src/__tests__/app/api/evaluation/types/route.test.ts@@ -0,0 +1,202 @@+jest.mock("@/lib/platform/evaluation/config", () => ({+ getEvaluationConfig: jest.fn(),+}));+jest.mock("@/lib/platform/evaluation/sync-rule-entities", () => ({+ syncRuleEntitiesFromConfig: jest.fn().mockResolvedValue(undefined),+}));+jest.mock("@/lib/posthog", () => ({+ __esModule: true,+ default: {+ fireEvent: jest.fn(),+ },+}));+jest.mock("@/constants/messages", () => ({+ __esModule: true,+ default: jest.fn(() => ({+ EVALUATION_THRESHOLD_SCORE_INVALID:+ "Threshold score must be a number between 0 and 1.",+ })),+}));+jest.mock("@/utils/json", () => ({+ jsonParse: jest.fn((v: string) => {+ try {+ return JSON.parse(v);+ } catch {+ return {};+ }+ }),+ jsonStringify: jest.fn((v: unknown) => JSON.stringify(v)),+}));+jest.mock("@/utils/asaw", () =>+ jest.fn(async (promise: Promise<any>) => {+ try {+ const result = await promise;+ return [null, result];+ } catch (err) {+ return [err, null];+ }+ })+);+jest.mock("@/lib/prisma", () => ({+ __esModule: true,+ default: {+ evaluationConfigs: {+ update: jest.fn().mockResolvedValue({}),+ },+ },+}));++import { GET, POST } from "@/app/api/evaluation/types/route";+import { getEvaluationConfig } from "@/lib/platform/evaluation/config";+import { syncRuleEntitiesFromConfig } from "@/lib/platform/evaluation/sync-rule-entities";+import prisma from "@/lib/prisma";++class TestResponse {+ status: number;+ private body: unknown;++ constructor(body?: unknown, init?: { status?: number }) {+ this.body = body;+ this.status = init?.status ?? 200;+ }++ static json(body: unknown, init?: { status?: number }) {+ return new TestResponse(body, init);+ }++ async json() {+ return this.body;+ }+}++(global as any).Response = TestResponse;++function makeRequest(body: Record<string, unknown>) {+ return {+ json: jest.fn().mockResolvedValue(body),+ } as unknown as Request;+}++function makeInvalidJsonRequest() {+ return {+ json: jest.fn().mockRejectedValue(new SyntaxError("Unexpected token")),+ } as unknown as Request;+}++describe("GET /api/evaluation/types", () => {+ beforeEach(() => {+ jest.clearAllMocks();+ });++ it("returns the configured evaluation types", async () => {+ (getEvaluationConfig as jest.Mock).mockResolvedValue({+ id: "cfg-1",+ evaluationTypes: [{ id: "hallucination", enabled: true }],+ });++ const response = await GET(undefined as any);+ expect(response.status).toBe(200);+ const data = await response.json();+ expect(data.data).toEqual([{ id: "hallucination", enabled: true }]);+ });++ it("returns empty data with an error when config is not found", async () => {+ (getEvaluationConfig as jest.Mock).mockRejectedValue(new Error("not found"));++ const response = await GET(undefined as any);+ expect(response.status).toBe(200);+ const data = await response.json();+ expect(data.data).toEqual([]);+ expect(data.err).toBeDefined();+ });+});++describe("POST /api/evaluation/types", () => {+ beforeEach(() => {+ jest.clearAllMocks();+ });++ it("returns 400 on invalid JSON body", async () => {+ const response = await POST(makeInvalidJsonRequest() as any);+ expect(response.status).toBe(400);+ const data = await response.json();+ expect(data.err).toBe("Invalid JSON body");+ });++ it("returns 400 when types is not an array", async () => {+ const response = await POST(makeRequest({ types: "not-an-array" }) as any);+ expect(response.status).toBe(400);+ const data = await response.json();+ expect(data.err).toBe("Invalid types array");+ });++ it("rejects a non-numeric thresholdScore without calling getEvaluationConfig", async () => {+ const response = await POST(+ makeRequest({+ types: [{ id: "toxicity", enabled: true, thresholdScore: "not-a-number" }],+ }) as any+ );++ expect(response.status).toBe(400);+ const data = await response.json();+ expect(data.err).toBe("Threshold score must be a number between 0 and 1.");+ expect(getEvaluationConfig).not.toHaveBeenCalled();+ });++ it("returns 400 when evaluation config is not found", async () => {+ (getEvaluationConfig as jest.Mock).mockRejectedValue(new Error("not found"));++ const response = await POST(+ makeRequest({ types: [{ id: "hallucination", enabled: true }] }) as any+ );++ expect(response.status).toBe(400);+ const data = await response.json();+ expect(data.err).toBe("Evaluation config not found");+ });++ it("clamps out-of-range thresholdScore values into [0, 1] and persists them", async () => {+ (getEvaluationConfig as jest.Mock).mockResolvedValue({+ id: "cfg-1",+ meta: "{}",+ });++ const response = await POST(+ makeRequest({+ types: [+ { id: "toxicity", enabled: true, thresholdScore: 1.5 },+ { id: "bias", enabled: true, thresholdScore: -0.2 },+ ],+ }) as any+ );++ expect(response.status).toBe(200);+ const data = await response.json();+ expect(data.data).toEqual([+ expect.objectContaining({ id: "toxicity", thresholdScore: 1 }),+ expect.objectContaining({ id: "bias", thresholdScore: 0 }),+ ]);+ const updateCall = (prisma.evaluationConfigs.update as jest.Mock).mock.calls[0][0];+ expect(updateCall.where).toEqual({ id: "cfg-1" });+ const persistedMeta = JSON.parse(updateCall.data.meta);+ expect(persistedMeta.evaluationTypes).toEqual([+ expect.objectContaining({ id: "toxicity", thresholdScore: 1 }),+ expect.objectContaining({ id: "bias", thresholdScore: 0 }),+ ]);+ expect(syncRuleEntitiesFromConfig).toHaveBeenCalled();+ });++ it("omits thresholdScore when not provided", async () => {+ (getEvaluationConfig as jest.Mock).mockResolvedValue({+ id: "cfg-1",+ meta: "{}",+ });++ const response = await POST(+ makeRequest({ types: [{ id: "hallucination", enabled: true }] }) as any+ );++ const data = await response.json();+ expect(data.data[0]).not.toHaveProperty("thresholdScore");+ });+});diff --git a/src/client/src/__tests__/lib/platform/evaluation/evaluation-config.test.ts b/src/client/src/__tests__/lib/platform/evaluation/evaluation-config.test.tsindex 98e3169b..c20a653d 100644--- a/src/client/src/__tests__/lib/platform/evaluation/evaluation-config.test.ts+++ b/src/client/src/__tests__/lib/platform/evaluation/evaluation-config.test.ts@@ -65,6 +65,7 @@ jest.mock('path', () => ({}));import { getEvaluationConfig, setEvaluationConfig, getEvaluationConfigById, restoreEvaluationCronJobs } from '@/lib/platform/evaluation/config';+import { normalizeThresholdScore } from '@/lib/platform/evaluation/threshold';import { getEvaluationTypeDefaultPrompts } from '@/lib/platform/evaluation/evaluation-type-defaults';import { getDBConfigByUser } from '@/lib/db-config';import prisma from '@/lib/prisma';@@ -766,6 +767,79 @@ describe('buildEvaluationTypesWithPrompts — custom evaluation types', () => {expect(hallucination!.rules).toHaveLength(1);expect(hallucination!.rules![0].ruleId).toBe('rule-hal-1');});++ it('built-in type overrides carry a configured thresholdScore through', async () => {+ const config = makeConfigWithMeta([+ { id: 'toxicity', enabled: true, thresholdScore: 0.75 },+ ]);+ (asaw as jest.Mock).mockResolvedValueOnce([null, config]);++ const result = await getEvaluationConfigById('eval-cfg-1');++ const toxicity = result.evaluationTypes!.find((t) => t.id === 'toxicity');+ expect(toxicity!.thresholdScore).toBe(0.75);+ });++ it('built-in types default to an undefined thresholdScore when not overridden', async () => {+ (asaw as jest.Mock).mockResolvedValueOnce([null, makeMockEvalConfig()]);++ const result = await getEvaluationConfigById('eval-cfg-1');++ const hallucination = result.evaluationTypes!.find((t) => t.id === 'hallucination');+ expect(hallucination!.thresholdScore).toBeUndefined();+ });++ it('custom types carry a configured thresholdScore through', async () => {+ const config = makeConfigWithMeta([+ { id: 'brand_voice', enabled: true, label: 'Brand Voice', thresholdScore: 0.3 },+ ]);+ (asaw as jest.Mock).mockResolvedValueOnce([null, config]);++ const result = await getEvaluationConfigById('eval-cfg-1');++ const brandVoice = result.evaluationTypes!.find((t) => t.id === 'brand_voice');+ expect(brandVoice!.thresholdScore).toBe(0.3);+ });++ it('ignores a non-numeric thresholdScore stored in meta and leaves it undefined', async () => {+ const config = makeConfigWithMeta([+ { id: 'toxicity', enabled: true, thresholdScore: 'not-a-number' as any },+ ]);+ (asaw as jest.Mock).mockResolvedValueOnce([null, config]);++ const result = await getEvaluationConfigById('eval-cfg-1');++ const toxicity = result.evaluationTypes!.find((t) => t.id === 'toxicity');+ expect(toxicity!.thresholdScore).toBeUndefined();+ });+});++describe('normalizeThresholdScore', () => {+ it('returns undefined for undefined, null, and empty string', () => {+ expect(normalizeThresholdScore(undefined)).toBeUndefined();+ expect(normalizeThresholdScore(null)).toBeUndefined();+ expect(normalizeThresholdScore('')).toBeUndefined();+ });++ it('returns NaN for a non-numeric value', () => {+ expect(Number.isNaN(normalizeThresholdScore('not-a-number'))).toBe(true);+ });++ it('passes through an in-range number unchanged', () => {+ expect(normalizeThresholdScore(0.42)).toBe(0.42);+ });++ it('clamps values above 1 down to 1', () => {+ expect(normalizeThresholdScore(1.5)).toBe(1);+ });++ it('clamps values below 0 up to 0', () => {+ expect(normalizeThresholdScore(-0.5)).toBe(0);+ });++ it('accepts numeric strings', () => {+ expect(normalizeThresholdScore('0.6')).toBe(0.6);+ });});describe('restoreEvaluationCronJobs', () => {diff --git a/src/client/src/__tests__/lib/platform/evaluation/evaluation.test.ts b/src/client/src/__tests__/lib/platform/evaluation/evaluation.test.tsindex e9260920..fca05de7 100644--- a/src/client/src/__tests__/lib/platform/evaluation/evaluation.test.ts+++ b/src/client/src/__tests__/lib/platform/evaluation/evaluation.test.ts@@ -60,6 +60,8 @@ jest.mock('@/helpers/server/trace', () => ({jest.mock('@/lib/platform/evaluation/rule-engine-context', () => ({getContextFromRuleEngineForTrace: jest.fn().mockResolvedValue({ contextContents: [], matchingRuleIds: [], contextEntityIds: [] }),getContextFromRulesWithPriority: jest.fn().mockResolvedValue({ contextContents: [], matchingRuleIds: [], contextEntityIds: [] }),+ getContextFromFields: jest.fn().mockResolvedValue({ contextContents: [], matchingRuleIds: [], contextEntityIds: [] }),+ getContextFromRulesWithPriorityForFields: jest.fn().mockResolvedValue({ contextContents: [], matchingRuleIds: [], contextEntityIds: [] }),}));jest.mock('@/constants/traces', () => ({SUPPORTED_EVALUATION_OPERATIONS: ['llm', 'chat'],@@ -75,7 +77,7 @@ jest.mock('@/lib/platform/evaluation/evaluation-type-defaults', () => ({getEvaluationTypeDefaultPrompt: jest.fn().mockResolvedValue(undefined),}));-import { getEvaluationsForSpanId, getEvaluationDetectedByType, autoEvaluate, setEvaluationsForSpanId, getEvaluationSummaryForSpanId, storeManualFeedback } from '@/lib/platform/evaluation/index';+import { getEvaluationsForSpanId, getEvaluationDetectedByType, autoEvaluate, setEvaluationsForSpanId, getEvaluationSummaryForSpanId, storeManualFeedback, runOfflineEvaluation } from '@/lib/platform/evaluation/index';import { dataCollector } from '@/lib/platform/common';import { getCurrentUser } from '@/lib/session';import { getEvaluationConfig, getEvaluationConfigById } from '@/lib/platform/evaluation/config';@@ -829,4 +831,395 @@ describe('setEvaluationsForSpanId — evaluationTypes branches', () => {'db-1');});++ it('applies per-type thresholdScore when storing dashboard/manual evaluation verdicts', async () => {+ (getRequestViaSpanId as jest.Mock).mockResolvedValue({+ record: { SpanId: 'span-1', SpanAttributes: {} },+ });+ (getEvaluationConfig as jest.Mock).mockResolvedValue({+ id: 'cfg-1',+ provider: 'openai',+ model: 'gpt-4',+ secret: { value: 'sk-1' },+ databaseConfigId: 'db-1',+ evaluationTypes: [+ { id: 'toxicity', enabled: true, label: 'Toxicity', thresholdScore: 0.8 },+ ],+ });+ // Raw LLM verdict would be "yes" under the default 0.5 threshold; the+ // per-type threshold of 0.8 must recompute it to "no" before storage.+ (runEvaluation as jest.Mock).mockResolvedValue({+ success: true,+ result: [+ {+ evaluation: 'toxicity',+ score: 0.6,+ verdict: 'yes',+ classification: 'mild',+ explanation: 'borderline',+ },+ ],+ });+ (dataCollector as jest.Mock).mockResolvedValue({ data: true, err: null });++ const result = await setEvaluationsForSpanId('span-1');+ expect(result).toMatchObject({ success: true });++ const insertCall = (dataCollector as jest.Mock).mock.calls.find(+ (call) => call[1] === 'insert'+ );+ expect(insertCall).toBeDefined();+ const stored = insertCall![0].values[0];+ // storeEvaluation remaps ids to configured labels (main normalizeEvaluationResults).+ expect(stored['evaluationData.evaluation']).toEqual(['Toxicity']);+ expect(stored['evaluationData.verdict']).toEqual(['no']);+ expect(stored.scores).toEqual({ Toxicity: 0.6 });+ });+});++describe('runOfflineEvaluation', () => {+ const baseConfig = {+ id: 'cfg-1',+ provider: 'openai',+ model: 'gpt-4',+ secret: { value: 'sk-1' },+ databaseConfigId: 'db-1',+ };++ beforeEach(() => {+ const { getContextFromFields, getContextFromRulesWithPriorityForFields } =+ require('@/lib/platform/evaluation/rule-engine-context');+ (getContextFromFields as jest.Mock).mockResolvedValue({ contextContents: [], matchingRuleIds: [], contextEntityIds: [] });+ (getContextFromRulesWithPriorityForFields as jest.Mock).mockResolvedValue({ contextContents: [], matchingRuleIds: [], contextEntityIds: [] });+ });++ it('rejects unknown eval types without calling runEvaluation', async () => {+ const config = {+ ...baseConfig,+ evaluationTypes: [{ id: 'hallucination', enabled: true, label: 'Hallucination' }],+ };++ const result = await runOfflineEvaluation(+ { prompt: 'p', response: 'r', evalTypes: ['bogus'] },+ config as any,+ 'db-1'+ );++ expect(result.success).toBe(false);+ expect(result.error).toBe('Unknown eval types: bogus');+ expect(runEvaluation).not.toHaveBeenCalled();+ });++ it('rejects explicitly requested but disabled eval types without calling runEvaluation', async () => {+ const config = {+ ...baseConfig,+ evaluationTypes: [+ { id: 'hallucination', enabled: true, label: 'Hallucination' },+ { id: 'bias', enabled: false, label: 'Bias' },+ ],+ };++ const result = await runOfflineEvaluation(+ { prompt: 'p', response: 'r', evalTypes: ['bias'] },+ config as any,+ 'db-1'+ );++ expect(result.success).toBe(false);+ expect(result.error).toBe('Disabled eval types: bias');+ expect(runEvaluation).not.toHaveBeenCalled();+ });++ it('reports unknown types even when other requested types are valid', async () => {+ const config = {+ ...baseConfig,+ evaluationTypes: [{ id: 'hallucination', enabled: true, label: 'Hallucination' }],+ };++ const result = await runOfflineEvaluation(+ { prompt: 'p', response: 'r', evalTypes: ['hallucination', 'bogus'] },+ config as any,+ 'db-1'+ );++ expect(result.success).toBe(false);+ expect(result.error).toBe('Unknown eval types: bogus');+ expect(runEvaluation).not.toHaveBeenCalled();+ });++ it('runs only the requested enabled types, excluding others from context', async () => {+ const config = {+ ...baseConfig,+ evaluationTypes: [+ { id: 'hallucination', enabled: true, label: 'Hallucination', defaultPrompt: 'hallucination ctx' },+ { id: 'bias', enabled: true, label: 'Bias', defaultPrompt: 'bias ctx' },+ ],+ };+ (runEvaluation as jest.Mock).mockResolvedValue({+ success: true,+ result: [{ evaluation: 'Hallucination', score: 0.1, classification: 'none', explanation: 'ok', verdict: 'no' }],+ });++ const result = await runOfflineEvaluation(+ { prompt: 'p', response: 'r', evalTypes: ['hallucination'] },+ config as any,+ 'db-1'+ );++ expect(result.success).toBe(true);+ expect(result.metadata?.evalTypesRun).toEqual(['hallucination']);+ const [{ contexts }] = (runEvaluation as jest.Mock).mock.calls[0];+ expect(contexts).toContain('hallucination ctx');+ expect(contexts).not.toContain('bias ctx');+ });++ it('defaults to all enabled types when eval_types is omitted', async () => {+ const config = {+ ...baseConfig,+ evaluationTypes: [+ { id: 'hallucination', enabled: true, label: 'Hallucination' },+ { id: 'bias', enabled: false, label: 'Bias' },+ { id: 'relevance', enabled: true, label: 'Relevance' },+ ],+ };+ (runEvaluation as jest.Mock).mockResolvedValue({ success: true, result: [] });++ const result = await runOfflineEvaluation(+ { prompt: 'p', response: 'r' },+ config as any,+ 'db-1'+ );++ expect(result.success).toBe(true);+ expect([...(result.metadata?.evalTypesRun || [])].sort()).toEqual(['hallucination', 'relevance']);+ });++ it('falls back to hallucination/bias/toxicity defaults when nothing is enabled and no eval_types requested', async () => {+ const config = {+ ...baseConfig,+ evaluationTypes: [+ { id: 'hallucination', enabled: false, label: 'Hallucination' },+ { id: 'bias', enabled: false, label: 'Bias' },+ { id: 'toxicity', enabled: false, label: 'Toxicity' },+ { id: 'relevance', enabled: false, label: 'Relevance' },+ ],+ };+ (runEvaluation as jest.Mock).mockResolvedValue({ success: true, result: [] });++ const result = await runOfflineEvaluation(+ { prompt: 'p', response: 'r' },+ config as any,+ 'db-1'+ );++ expect([...(result.metadata?.evalTypesRun || [])].sort()).toEqual(['bias', 'hallucination', 'toxicity']);+ });++ it('overrides verdict to "no" when the per-type threshold is stricter than the raw score', async () => {+ const config = {+ ...baseConfig,+ evaluationTypes: [{ id: 'toxicity', enabled: true, label: 'Toxicity', thresholdScore: 0.8 }],+ };+ // Score of 0.6 would be 'yes' under the default 0.5 threshold, but the+ // per-type threshold of 0.8 should keep it 'no'.+ (runEvaluation as jest.Mock).mockResolvedValue({+ success: true,+ result: [{ evaluation: 'Toxicity', score: 0.6, classification: 'mild', explanation: 'x', verdict: 'yes' }],+ });++ const result = await runOfflineEvaluation(+ { prompt: 'p', response: 'r', evalTypes: ['toxicity'] },+ config as any,+ 'db-1'+ );++ expect(result.evaluations?.[0].verdict).toBe('no');+ });++ it('overrides verdict to "yes" when the score exceeds a stricter per-type threshold', async () => {+ const config = {+ ...baseConfig,+ evaluationTypes: [{ id: 'toxicity', enabled: true, label: 'Toxicity', thresholdScore: 0.3 }],+ };+ (runEvaluation as jest.Mock).mockResolvedValue({+ success: true,+ result: [{ evaluation: 'Toxicity', score: 0.5, classification: 'mild', explanation: 'x', verdict: 'no' }],+ });++ const result = await runOfflineEvaluation(+ { prompt: 'p', response: 'r', evalTypes: ['toxicity'] },+ config as any,+ 'db-1'+ );++ expect(result.evaluations?.[0].verdict).toBe('yes');+ });++ it('falls back to the request thresholdScore when the type has no per-type threshold configured', async () => {+ const config = {+ ...baseConfig,+ evaluationTypes: [{ id: 'toxicity', enabled: true, label: 'Toxicity' }],+ };+ (runEvaluation as jest.Mock).mockResolvedValue({+ success: true,+ result: [{ evaluation: 'Toxicity', score: 0.9, classification: 'severe', explanation: 'x', verdict: 'no' }],+ });++ const result = await runOfflineEvaluation(+ { prompt: 'p', response: 'r', evalTypes: ['toxicity'], thresholdScore: 0.95 },+ config as any,+ 'db-1'+ );++ // 0.9 does not exceed the request-level 0.95 threshold+ expect(result.evaluations?.[0].verdict).toBe('no');+ });++ it('falls back to the request threshold when the returned evaluation label matches no configured type', async () => {+ const config = {+ ...baseConfig,+ evaluationTypes: [+ { id: 'sensitivity', enabled: true, label: 'PII - Financial', thresholdScore: 0.9 },+ ],+ };+ // The model echoes back a label that doesn't match the configured id or+ // label exactly (e.g. a custom prompt without the matching context header).+ (runEvaluation as jest.Mock).mockResolvedValue({+ success: true,+ result: [{ evaluation: 'Pii Financial Data', score: 0.6, classification: 'leak', explanation: 'x', verdict: 'yes' }],+ });++ const result = await runOfflineEvaluation(+ { prompt: 'p', response: 'r', evalTypes: ['sensitivity'], thresholdScore: 0.5 },+ config as any,+ 'db-1'+ );++ // The unmatched result still comes back (no crash, no dropped row), and+ // since its label didn't match, the 0.9 per-type threshold is NOT applied —+ // it falls back to the request-level 0.5 threshold instead.+ expect(result.success).toBe(true);+ expect(result.evaluations).toHaveLength(1);+ expect(result.evaluations?.[0].verdict).toBe('yes');+ });++ it('returns failure when runEvaluation fails', async () => {+ const config = {+ ...baseConfig,+ evaluationTypes: [{ id: 'hallucination', enabled: true, label: 'Hallucination' }],+ };+ (runEvaluation as jest.Mock).mockResolvedValue({ success: false, error: 'Model error' });++ const result = await runOfflineEvaluation(+ { prompt: 'p', response: 'r' },+ config as any,+ 'db-1'+ );++ expect(result.success).toBe(false);+ expect(result.error).toBe('Model error');+ });++ describe('rule-priority context consistency with the real-time path', () => {+ it('collects rules from enabled types and resolves context via getContextFromRulesWithPriorityForFields, matching the real-time path', async () => {+ const { getContextFromRulesWithPriorityForFields, getContextFromFields } =+ require('@/lib/platform/evaluation/rule-engine-context');+ (getContextFromRulesWithPriorityForFields as jest.Mock).mockResolvedValue({+ contextContents: ['linked rule context'],+ matchingRuleIds: ['r1'],+ contextEntityIds: ['e1'],+ });+ const config = {+ ...baseConfig,+ evaluationTypes: [+ { id: 'hallucination', enabled: true, rules: [{ ruleId: 'r1', priority: 5 }] },+ ],+ };+ (runEvaluation as jest.Mock).mockResolvedValue({ success: true, result: [] });++ const result = await runOfflineEvaluation(+ { prompt: 'p', response: 'r', attributes: { 'service.name': 'my-app' } },+ config as any,+ 'db-1'+ );++ expect(getContextFromRulesWithPriorityForFields).toHaveBeenCalledWith(+ { 'service.name': 'my-app' },+ [{ ruleId: 'r1', priority: 5 }],+ 'db-1'+ );+ expect(getContextFromFields).not.toHaveBeenCalled();+ expect(result.contextApplied?.matchingRuleIds).toEqual(['r1']);+ });++ it('collects legacy ruleId/priority fields into rulesWithPriority, same as the real-time path', async () => {+ const { getContextFromRulesWithPriorityForFields } =+ require('@/lib/platform/evaluation/rule-engine-context');+ (getContextFromRulesWithPriorityForFields as jest.Mock).mockResolvedValue({+ contextContents: [], matchingRuleIds: [], contextEntityIds: [],+ });+ const config = {+ ...baseConfig,+ evaluationTypes: [+ { id: 'hallucination', enabled: true, ruleId: 'r-legacy', priority: 2 },+ ],+ };+ (runEvaluation as jest.Mock).mockResolvedValue({ success: true, result: [] });++ await runOfflineEvaluation(+ { prompt: 'p', response: 'r', attributes: { 'service.name': 'my-app' } },+ config as any,+ 'db-1'+ );++ expect(getContextFromRulesWithPriorityForFields).toHaveBeenCalledWith(+ { 'service.name': 'my-app' },+ [{ ruleId: 'r-legacy', priority: 2 }],+ 'db-1'+ );+ });++ it('falls back to getContextFromFields when no enabled type has linked rules', async () => {+ const { getContextFromFields, getContextFromRulesWithPriorityForFields } =+ require('@/lib/platform/evaluation/rule-engine-context');+ (getContextFromFields as jest.Mock).mockResolvedValue({+ contextContents: ['generic context'], matchingRuleIds: [], contextEntityIds: [],+ });+ const config = {+ ...baseConfig,+ evaluationTypes: [{ id: 'hallucination', enabled: true }],+ };+ (runEvaluation as jest.Mock).mockResolvedValue({ success: true, result: [] });++ await runOfflineEvaluation(+ { prompt: 'p', response: 'r', attributes: { 'service.name': 'my-app' } },+ config as any,+ 'db-1'+ );++ expect(getContextFromFields).toHaveBeenCalledWith({ 'service.name': 'my-app' }, 'db-1');+ expect(getContextFromRulesWithPriorityForFields).not.toHaveBeenCalled();+ });++ it('does not call the rule engine at all when no attributes are provided', async () => {+ const { getContextFromFields, getContextFromRulesWithPriorityForFields } =+ require('@/lib/platform/evaluation/rule-engine-context');+ const config = {+ ...baseConfig,+ evaluationTypes: [+ { id: 'hallucination', enabled: true, rules: [{ ruleId: 'r1', priority: 5 }] },+ ],+ };+ (runEvaluation as jest.Mock).mockResolvedValue({ success: true, result: [] });++ await runOfflineEvaluation(+ { prompt: 'p', response: 'r' },+ config as any,+ 'db-1'+ );++ expect(getContextFromFields).not.toHaveBeenCalled();+ expect(getContextFromRulesWithPriorityForFields).not.toHaveBeenCalled();+ });+ });});diff --git a/src/client/src/__tests__/lib/platform/evaluation/offline-auth.test.ts b/src/client/src/__tests__/lib/platform/evaluation/offline-auth.test.tsnew file mode 100644index 00000000..037ff4e3--- /dev/null+++ b/src/client/src/__tests__/lib/platform/evaluation/offline-auth.test.ts@@ -0,0 +1,152 @@+jest.mock('@/lib/platform/api-keys', () => ({+ getAPIKeyInfo: jest.fn(),+}));+jest.mock('@/lib/platform/evaluation/config', () => ({+ getEvaluationConfigByDbConfigId: jest.fn(),+}));+jest.mock('@/constants/messages', () => ({+ __esModule: true,+ default: jest.fn(() => ({+ NO_API_KEY: 'No API key provided',+ })),+}));+jest.mock('@/utils/asaw', () =>+ jest.fn(async (promise: Promise<any>) => {+ try {+ const result = await promise;+ return [null, result];+ } catch (err) {+ return [err, null];+ }+ })+);++import {+ authenticateOfflineApiKey,+ loadOfflineEvaluationConfig,+ EVALUATION_NOT_CONFIGURED_MESSAGE,+} from '@/lib/platform/evaluation/offline-auth';+import { getAPIKeyInfo } from '@/lib/platform/api-keys';+import { getEvaluationConfigByDbConfigId } from '@/lib/platform/evaluation/config';++class TestResponse {+ status: number;+ private body: unknown;++ constructor(body?: unknown, init?: { status?: number }) {+ this.body = body;+ this.status = init?.status ?? 200;+ }++ static json(body: unknown, init?: { status?: number }) {+ return new TestResponse(body, init);+ }++ async json() {+ return this.body;+ }+}++(global as any).Response = TestResponse;++function makeRequest(authorization?: string) {+ return {+ headers: {+ get: (key: string) => (key.toLowerCase() === 'authorization' ? authorization ?? null : null),+ },+ } as unknown as Request;+}++beforeEach(() => {+ jest.clearAllMocks();+});++describe('authenticateOfflineApiKey', () => {+ it('returns a 401 error when no Authorization header is present', async () => {+ const result = await authenticateOfflineApiKey(makeRequest());+ expect('error' in result).toBe(true);+ if ('error' in result) {+ expect(result.error.status).toBe(401);+ }+ expect(getAPIKeyInfo).not.toHaveBeenCalled();+ });++ it('returns a 401 error when the Authorization header is not a Bearer token', async () => {+ const result = await authenticateOfflineApiKey(makeRequest('Basic abc123'));+ expect('error' in result).toBe(true);+ expect(getAPIKeyInfo).not.toHaveBeenCalled();+ });++ it('returns a 401 error when the Bearer token is empty', async () => {+ const result = await authenticateOfflineApiKey(makeRequest('Bearer '));+ expect('error' in result).toBe(true);+ expect(getAPIKeyInfo).not.toHaveBeenCalled();+ });++ it('returns a 401 error when the API key is not found', async () => {+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([new Error('not found'), null]);+ const result = await authenticateOfflineApiKey(makeRequest('Bearer openlit-key'));+ expect('error' in result).toBe(true);+ if ('error' in result) {+ expect(result.error.status).toBe(401);+ }+ });++ it('returns a 401 error when the API key has no databaseConfigId', async () => {+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([null, {}]);+ const result = await authenticateOfflineApiKey(makeRequest('Bearer openlit-key'));+ expect('error' in result).toBe(true);+ });++ it('returns the databaseConfigId for a valid API key', async () => {+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([null, { databaseConfigId: 'db-1' }]);+ const result = await authenticateOfflineApiKey(makeRequest('Bearer openlit-key'));+ expect('error' in result).toBe(false);+ if (!('error' in result)) {+ expect(result.databaseConfigId).toBe('db-1');+ }+ });+});++describe('loadOfflineEvaluationConfig', () => {+ it('calls the caller-provided notConfiguredResponse when the config lookup fails', async () => {+ (getEvaluationConfigByDbConfigId as jest.Mock).mockRejectedValue(new Error('not found'));+ const notConfiguredResponse = jest.fn(() => new Response(null, { status: 400 }) as any);++ const result = await loadOfflineEvaluationConfig('db-1', notConfiguredResponse);++ expect('error' in result).toBe(true);+ expect(notConfiguredResponse).toHaveBeenCalledTimes(1);+ });++ it('calls the caller-provided notConfiguredResponse when the config has no id', async () => {+ (getEvaluationConfigByDbConfigId as jest.Mock).mockResolvedValue({});+ const notConfiguredResponse = jest.fn(() => new Response(null, { status: 400 }) as any);++ const result = await loadOfflineEvaluationConfig('db-1', notConfiguredResponse);++ expect('error' in result).toBe(true);+ expect(notConfiguredResponse).toHaveBeenCalledTimes(1);+ });++ it('returns the config when found, without calling notConfiguredResponse', async () => {+ (getEvaluationConfigByDbConfigId as jest.Mock).mockResolvedValue({ id: 'cfg-1' });+ const notConfiguredResponse = jest.fn(() => new Response(null, { status: 400 }) as any);++ const result = await loadOfflineEvaluationConfig('db-1', notConfiguredResponse);++ expect('error' in result).toBe(false);+ if (!('error' in result)) {+ expect(result.config.id).toBe('cfg-1');+ }+ expect(notConfiguredResponse).not.toHaveBeenCalled();+ expect(getEvaluationConfigByDbConfigId).toHaveBeenCalledWith('db-1', true);+ });+});++describe('EVALUATION_NOT_CONFIGURED_MESSAGE', () => {+ it('is a non-empty string', () => {+ expect(typeof EVALUATION_NOT_CONFIGURED_MESSAGE).toBe('string');+ expect(EVALUATION_NOT_CONFIGURED_MESSAGE.length).toBeGreaterThan(0);+ });+});diff --git a/src/client/src/__tests__/lib/platform/evaluation/rule-engine-context.test.ts b/src/client/src/__tests__/lib/platform/evaluation/rule-engine-context.test.tsindex d1b0a7eb..2a2bd8b8 100644--- a/src/client/src/__tests__/lib/platform/evaluation/rule-engine-context.test.ts+++ b/src/client/src/__tests__/lib/platform/evaluation/rule-engine-context.test.ts@@ -6,6 +6,8 @@ import {extractRuleEngineFieldsFromTrace,getContextFromRuleEngineForTrace,getContextFromRulesWithPriority,+ getContextFromFields,+ getContextFromRulesWithPriorityForFields,} from '@/lib/platform/evaluation/rule-engine-context';import { evaluateRules } from '@/lib/platform/rule-engine/evaluate';@@ -231,3 +233,105 @@ describe('getContextFromRulesWithPriority', () => {});});++// getContextFromFields / getContextFromRulesWithPriorityForFields are the+// shared, trace-independent primitives behind the trace-based functions+// above — used directly by the offline/SDK evaluation path so it resolves+// rule-engine context identically to the real-time/auto path for the same+// input fields.+describe('getContextFromFields', () => {+ it('returns empty result and skips evaluateRules when fields is empty', async () => {+ const result = await getContextFromFields({});+ expect(result).toEqual({ contextContents: [], matchingRuleIds: [], contextEntityIds: [] });+ expect(evaluateRules).not.toHaveBeenCalled();+ });++ it('calls evaluateRules with the given fields', async () => {+ await getContextFromFields({ 'service.name': 'my-app' }, 'db-1');+ expect(evaluateRules).toHaveBeenCalledWith(+ { fields: { 'service.name': 'my-app' }, entity_type: 'context', include_entity_data: true },+ 'db-1'+ );+ });++ it('extracts context content from entity_data', async () => {+ (evaluateRules as jest.Mock).mockResolvedValue({+ matchingRuleIds: ['r1'],+ entities: [{ rule_id: 'r1', entity_type: 'context', entity_id: 'ctx-1' }],+ entity_data: { 'context:ctx-1': { content: 'field-based content' } },+ });+ const result = await getContextFromFields({ 'service.name': 'my-app' });+ expect(result.contextContents).toEqual(['field-based content']);+ expect(result.contextEntityIds).toEqual(['ctx-1']);+ expect(result.matchingRuleIds).toEqual(['r1']);+ });++ it('returns empty result when evaluateRules throws', async () => {+ (evaluateRules as jest.Mock).mockRejectedValue(new Error('boom'));+ const result = await getContextFromFields({ 'service.name': 'my-app' });+ expect(result).toEqual({ contextContents: [], matchingRuleIds: [], contextEntityIds: [] });+ });+});++describe('getContextFromRulesWithPriorityForFields', () => {+ it('returns raw result when rulesWithPriority is empty', async () => {+ (evaluateRules as jest.Mock).mockResolvedValue({+ matchingRuleIds: ['r1'],+ entities: [],+ entity_data: { 'context:ctx-1': { content: 'base content' } },+ });+ const result = await getContextFromRulesWithPriorityForFields({ 'service.name': 'my-app' }, []);+ expect(result.matchingRuleIds).toContain('r1');+ });++ it('orders context by priority across the given fields, same as the trace-based path', async () => {+ (evaluateRules as jest.Mock)+ .mockResolvedValueOnce({ matchingRuleIds: ['r1', 'r2'], entities: [], entity_data: {} })+ .mockResolvedValueOnce({+ matchingRuleIds: ['r1', 'r2'],+ entities: [+ { rule_id: 'r1', entity_type: 'context', entity_id: 'ctx-1' },+ { rule_id: 'r2', entity_type: 'context', entity_id: 'ctx-2' },+ ],+ entity_data: {+ 'context:ctx-1': { content: 'low priority content' },+ 'context:ctx-2': { content: 'high priority content' },+ },+ });++ const result = await getContextFromRulesWithPriorityForFields(+ { 'service.name': 'my-app' },+ [+ { ruleId: 'r1', priority: 1 },+ { ruleId: 'r2', priority: 10 },+ ]+ );++ expect(result.contextContents[0]).toBe('high priority content');+ expect(result.contextContents[1]).toBe('low priority content');+ expect(result.matchingRuleIds).toEqual(['r2', 'r1']);+ });++ it('returns empty when fields is empty (even with priority rules)', async () => {+ const result = await getContextFromRulesWithPriorityForFields({}, [{ ruleId: 'r1', priority: 5 }]);+ expect(result).toEqual({ contextContents: [], matchingRuleIds: [], contextEntityIds: [] });+ });++ it('returns the original (unordered) result instead of throwing when the priority re-fetch fails', async () => {+ (evaluateRules as jest.Mock)+ .mockResolvedValueOnce({+ matchingRuleIds: ['r1'],+ entities: [],+ entity_data: { 'context:ctx-x': { content: 'fallback content' } },+ })+ .mockRejectedValueOnce(new Error('second fetch failed'));++ const result = await getContextFromRulesWithPriorityForFields(+ { 'service.name': 'my-app' },+ [{ ruleId: 'r1', priority: 5 }]+ );++ expect(result.matchingRuleIds).toContain('r1');+ expect(result.contextContents).toEqual(['fallback content']);+ });+});diff --git a/src/client/src/__tests__/lib/platform/evaluation/type-config.test.ts b/src/client/src/__tests__/lib/platform/evaluation/type-config.test.tsnew file mode 100644index 00000000..60ab0992--- /dev/null+++ b/src/client/src/__tests__/lib/platform/evaluation/type-config.test.ts@@ -0,0 +1,174 @@+jest.mock('@/lib/platform/evaluation/sync-rule-entities', () => ({+ syncRuleEntitiesFromConfig: jest.fn().mockResolvedValue(undefined),+}));+jest.mock('@/utils/json', () => ({+ jsonParse: jest.fn((v: string) => {+ try { return JSON.parse(v); } catch { return {}; }+ }),+ jsonStringify: jest.fn((v: unknown) => JSON.stringify(v)),+}));+jest.mock('@/lib/prisma', () => ({+ __esModule: true,+ default: {+ evaluationConfigs: {+ update: jest.fn().mockResolvedValue({}),+ },+ },+}));++import {+ normalizeRules,+ normalizeTypeConfig,+ mergeTypeIntoList,+ upsertEvaluationTypes,+ persistEvaluationTypes,+} from '@/lib/platform/evaluation/type-config';+import { syncRuleEntitiesFromConfig } from '@/lib/platform/evaluation/sync-rule-entities';+import prisma from '@/lib/prisma';++beforeEach(() => {+ jest.clearAllMocks();+});++describe('normalizeRules', () => {+ it('returns an empty array for a non-array input', () => {+ expect(normalizeRules('not-an-array' as any)).toEqual([]);+ });++ it('filters out entries without a ruleId', () => {+ const result = normalizeRules([{ ruleId: 'r1', priority: 2 }, { priority: 3 }, { ruleId: '' }]);+ expect(result).toEqual([{ ruleId: 'r1', priority: 2 }]);+ });++ it('defaults priority to 0 when missing or non-numeric', () => {+ const result = normalizeRules([{ ruleId: 'r1' }, { ruleId: 'r2', priority: 'bad' }]);+ expect(result).toEqual([{ ruleId: 'r1', priority: 0 }, { ruleId: 'r2', priority: 0 }]);+ });+});++describe('normalizeTypeConfig', () => {+ it('builds a minimal built-in type config', () => {+ const result = normalizeTypeConfig({ id: 'toxicity', enabled: true }, undefined);+ expect(result).toEqual({ id: 'toxicity', enabled: true, rules: [] });+ });++ it('includes thresholdScore only when provided', () => {+ const result = normalizeTypeConfig({ id: 'toxicity', enabled: true }, 0.8);+ expect(result.thresholdScore).toBe(0.8);+ const withoutThreshold = normalizeTypeConfig({ id: 'toxicity', enabled: true }, undefined);+ expect(withoutThreshold).not.toHaveProperty('thresholdScore');+ });++ it('collects rules from the rules array over the legacy ruleId field', () => {+ const result = normalizeTypeConfig(+ { id: 'toxicity', enabled: true, rules: [{ ruleId: 'r1', priority: 2 }], ruleId: 'legacy' },+ undefined+ );+ expect(result.rules).toEqual([{ ruleId: 'r1', priority: 2 }]);+ });++ it('falls back to the legacy ruleId/priority fields when rules is absent', () => {+ const result = normalizeTypeConfig({ id: 'toxicity', enabled: true, ruleId: 'r-legacy', priority: 3 }, undefined);+ expect(result.rules).toEqual([{ ruleId: 'r-legacy', priority: 3 }]);+ });++ it('preserves custom type metadata for isCustom types', () => {+ const result = normalizeTypeConfig(+ { id: 'my_custom', enabled: true, isCustom: true, label: ' Custom Label ', description: ' Desc ', prompt: ' Custom prompt ' },+ undefined+ );+ expect(result).toMatchObject({+ isCustom: true,+ label: 'Custom Label',+ description: 'Desc',+ prompt: 'Custom prompt',+ });+ });++ it('trims a custom prompt for built-in (non-custom) types too', () => {+ const result = normalizeTypeConfig({ id: 'toxicity', enabled: true, prompt: ' override ' }, undefined);+ expect(result.prompt).toBe('override');+ expect(result.isCustom).toBeUndefined();+ });+});++describe('mergeTypeIntoList', () => {+ it('appends a new entry when no existing entry has the same id', () => {+ const result = mergeTypeIntoList([{ id: 'a', enabled: true }], { id: 'b', enabled: false });+ expect(result).toEqual([{ id: 'a', enabled: true }, { id: 'b', enabled: false }]);+ });++ it('replaces the existing entry in place when the id already exists', () => {+ const result = mergeTypeIntoList(+ [{ id: 'a', enabled: true }, { id: 'b', enabled: false }],+ { id: 'a', enabled: false }+ );+ expect(result).toEqual([{ id: 'a', enabled: false }, { id: 'b', enabled: false }]);+ });++ it('does not mutate the original array', () => {+ const original = [{ id: 'a', enabled: true }];+ mergeTypeIntoList(original, { id: 'a', enabled: false });+ expect(original).toEqual([{ id: 'a', enabled: true }]);+ });+});++describe('upsertEvaluationTypes', () => {+ it('upserts incoming types without dropping unrelated existing entries', () => {+ const result = upsertEvaluationTypes(+ [+ { id: 'bias', enabled: true, thresholdScore: 0.4 },+ { id: 'toxicity', enabled: false },+ ],+ [{ id: 'toxicity', enabled: true, thresholdScore: 0.9 }]+ );+ expect(result).toEqual([+ { id: 'bias', enabled: true, thresholdScore: 0.4 },+ { id: 'toxicity', enabled: true, thresholdScore: 0.9 },+ ]);+ });++ it('appends types that are not already present', () => {+ const result = upsertEvaluationTypes(+ [{ id: 'bias', enabled: true }],+ [{ id: 'toxicity', enabled: true }]+ );+ expect(result).toEqual([+ { id: 'bias', enabled: true },+ { id: 'toxicity', enabled: true },+ ]);+ });+});++describe('persistEvaluationTypes', () => {+ it('merges evaluationTypes into the existing meta and persists it', async () => {+ await persistEvaluationTypes(+ 'cfg-1',+ JSON.stringify({ cronJobId: 'keep-me' }),+ [{ id: 'toxicity', enabled: true }]+ );++ expect(prisma.evaluationConfigs.update).toHaveBeenCalledWith({+ where: { id: 'cfg-1' },+ data: {+ meta: JSON.stringify({+ cronJobId: 'keep-me',+ evaluationTypes: [{ id: 'toxicity', enabled: true }],+ }),+ },+ });+ });++ it('handles a null/empty meta by starting fresh', async () => {+ await persistEvaluationTypes('cfg-1', null, [{ id: 'toxicity', enabled: true }]);+ expect(prisma.evaluationConfigs.update).toHaveBeenCalledWith({+ where: { id: 'cfg-1' },+ data: { meta: JSON.stringify({ evaluationTypes: [{ id: 'toxicity', enabled: true }] }) },+ });+ });++ it('re-syncs rule entities after persisting', async () => {+ await persistEvaluationTypes('cfg-1', '{}', []);+ expect(syncRuleEntitiesFromConfig).toHaveBeenCalled();+ });+});diff --git a/src/client/src/app/(playground)/evaluations/types/[id]/page.tsx b/src/client/src/app/(playground)/evaluations/types/[id]/page.tsxindex 43044b08..ac0ad0aa 100644--- a/src/client/src/app/(playground)/evaluations/types/[id]/page.tsx+++ b/src/client/src/app/(playground)/evaluations/types/[id]/page.tsx@@ -4,6 +4,8 @@ import { useParams, useRouter } from "next/navigation";import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";import { Badge } from "@/components/ui/badge";import { Switch } from "@/components/ui/switch";+import { Input } from "@/components/ui/input";+import { Label } from "@/components/ui/label";import {Select,SelectContent,@@ -38,6 +40,7 @@ interface EvaluationTypeConfig {rules: RuleWithPriority[];prompt?: string;defaultPrompt?: string;+ thresholdScore?: number;}export default function EvaluationTypeDetailPage() {@@ -94,6 +97,7 @@ export default function EvaluationTypeDetailPage() {rules: data.rules || [],prompt: data.prompt,defaultPrompt: data.defaultPrompt,+ thresholdScore: data.thresholdScore,});return;}@@ -106,6 +110,7 @@ export default function EvaluationTypeDetailPage() {rules: [],prompt: data?.prompt,defaultPrompt: data?.defaultPrompt,+ thresholdScore: data?.thresholdScore,});return;}@@ -163,6 +168,7 @@ export default function EvaluationTypeDetailPage() {enabled: config.enabled,rules: config.rules.filter((r) => r.ruleId),prompt: config.prompt?.trim() || undefined,+ thresholdScore: config.thresholdScore,};if (config.isCustom) {payload.isCustom = true;@@ -313,6 +319,62 @@ export default function EvaluationTypeDetailPage() {</Card>)}+ <Card className="border-stone-200 dark:border-stone-800 shadow-sm">+ <CardHeader className="pb-2">+ <CardTitle className="text-base">+ {getMessage().EVALUATION_TYPE_THRESHOLD_LABEL}+ </CardTitle>+ <p className="text-xs text-stone-500 dark:text-stone-400 font-normal">+ {getMessage().EVALUATION_TYPE_THRESHOLD_DESCRIPTION}+ </p>+ </CardHeader>+ <CardContent>+ <div className="max-w-[160px]">+ <Label className="sr-only">+ {getMessage().EVALUATION_TYPE_THRESHOLD_LABEL}+ </Label>+ <Input+ type="number"+ min={0}+ max={1}+ step={0.05}+ placeholder={+ getMessage().EVALUATION_TYPE_THRESHOLD_PLACEHOLDER+ }+ value={config?.thresholdScore ?? ""}+ onChange={(e) => {+ const raw = e.target.value;+ setConfig((prev) => {+ if (!prev) return prev;+ if (raw === "") {+ return { ...prev, thresholdScore: undefined };+ }+ const nextValue = Number(raw);+ // Ignore unparseable partial input (e.g. "-", "1..2",+ // "abc") instead of propagating NaN into config and+ // on to the API — keep the last valid value until the+ // user types something parseable.+ if (Number.isNaN(nextValue)) return prev;+ return { ...prev, thresholdScore: nextValue };+ });+ }}+ onBlur={(e) => {+ const raw = e.target.value;+ if (raw === "") return;+ const parsed = Number(raw);+ if (Number.isNaN(parsed)) return;+ const clamped = Math.min(1, Math.max(0, parsed));+ if (clamped !== parsed) {+ setConfig((prev) =>+ prev ? { ...prev, thresholdScore: clamped } : prev+ );+ }+ }}+ />+ </div>+ </CardContent>+ </Card>+<Card className="border-stone-200 dark:border-stone-800 shadow-sm"><CardHeader className="pb-3"><div className="flex items-start justify-between gap-4">diff --git a/src/client/src/app/(playground)/evaluations/types/new/page.tsx b/src/client/src/app/(playground)/evaluations/types/new/page.tsxindex ef405d41..e5451a5f 100644--- a/src/client/src/app/(playground)/evaluations/types/new/page.tsx+++ b/src/client/src/app/(playground)/evaluations/types/new/page.tsx@@ -69,6 +69,10 @@ export default function CreateCustomEvaluationTypePage() {description: t.isCustom ? t.description : undefined,prompt: t.prompt,rules: t.rules,+ // Preserve per-type thresholds across create-and-replace.+ // POST /api/evaluation/types replaces the full meta list;+ // omitting thresholdScore would wipe every saved override.+ thresholdScore: t.thresholdScore,})),{id,diff --git a/src/client/src/app/api/evaluation/offline/types/[id]/route.ts b/src/client/src/app/api/evaluation/offline/types/[id]/route.tsnew file mode 100644index 00000000..c046b20b--- /dev/null+++ b/src/client/src/app/api/evaluation/offline/types/[id]/route.ts@@ -0,0 +1,100 @@+import { SERVER_EVENTS } from "@/constants/events";+import {+ authenticateOfflineApiKey,+ loadOfflineEvaluationConfig,+ EVALUATION_NOT_CONFIGURED_MESSAGE,+} from "@/lib/platform/evaluation/offline-auth";+import { normalizeThresholdScore } from "@/lib/platform/evaluation/threshold";+import {+ normalizeRules,+ mergeTypeIntoList,+ persistEvaluationTypes,+ EvaluationTypeConfig,+} from "@/lib/platform/evaluation/type-config";+import getMessage from "@/constants/messages";+import PostHogServer from "@/lib/posthog";+import { jsonParse } from "@/utils/json";+import { errorResponse } from "@/helpers/server/response";++export async function PATCH(+ request: Request,+ { params }: { params: { id: string } }+) {+ const startTimestamp = Date.now();+ const typeId = params.id;++ const auth = await authenticateOfflineApiKey(request);+ if ("error" in auth) return auth.error;++ let body: any;+ try {+ body = await request.json();+ } catch {+ return errorResponse("Invalid JSON body", 400);+ }++ const thresholdScore =+ body.thresholdScore !== undefined+ ? normalizeThresholdScore(body.thresholdScore)+ : undefined;+ if (Number.isNaN(thresholdScore)) {+ PostHogServer.fireEvent({+ event: SERVER_EVENTS.EVALUATION_OFFLINE_TYPE_UPDATE_FAILURE,+ startTimestamp,+ });+ return errorResponse(getMessage().EVALUATION_THRESHOLD_SCORE_INVALID, 400);+ }++ const loaded = await loadOfflineEvaluationConfig(auth.databaseConfigId, () => {+ PostHogServer.fireEvent({+ event: SERVER_EVENTS.EVALUATION_OFFLINE_TYPE_UPDATE_FAILURE,+ startTimestamp,+ });+ return errorResponse(EVALUATION_NOT_CONFIGURED_MESSAGE, 400);+ });+ if ("error" in loaded) return loaded.error;++ const meta = jsonParse((loaded.config as any).meta || "{}") as Record<string, any>;+ const types: EvaluationTypeConfig[] =+ (meta.evaluationTypes as EvaluationTypeConfig[]) || [];+ const idx = types.findIndex((t: any) => t.id === typeId);+ const existing = idx >= 0 ? types[idx] : { id: typeId, enabled: false, rules: [] };+ const updated: EvaluationTypeConfig = {+ id: typeId,+ enabled: body.enabled ?? existing.enabled,+ rules: body.rules !== undefined ? normalizeRules(body.rules) : existing.rules || [],+ prompt: body.prompt !== undefined ? body.prompt : existing.prompt,+ thresholdScore:+ body.thresholdScore !== undefined+ ? thresholdScore+ : (existing as any).thresholdScore,+ };+ if (body.isCustom || (existing as any).isCustom) {+ updated.isCustom = true;+ updated.label = body.label ?? (existing as any).label ?? typeId;+ updated.description = body.description ?? (existing as any).description ?? "";+ }++ await persistEvaluationTypes(+ loaded.config.id,+ (loaded.config as any).meta,+ mergeTypeIntoList(types, updated)+ );++ PostHogServer.fireEvent({+ event: SERVER_EVENTS.EVALUATION_OFFLINE_TYPE_UPDATE_SUCCESS,+ startTimestamp,+ });+ return Response.json({ data: updated });+}++export async function OPTIONS() {+ return new Response(null, {+ status: 200,+ headers: {+ "Access-Control-Allow-Origin": "*",+ "Access-Control-Allow-Methods": "PATCH, OPTIONS",+ "Access-Control-Allow-Headers": "Content-Type, Authorization",+ },+ });+}diff --git a/src/client/src/app/api/evaluation/offline/types/route.ts b/src/client/src/app/api/evaluation/offline/types/route.tsindex fc038166..afbaec87 100644--- a/src/client/src/app/api/evaluation/offline/types/route.ts+++ b/src/client/src/app/api/evaluation/offline/types/route.ts@@ -1,51 +1,130 @@-import { getAPIKeyInfo } from "@/lib/platform/api-keys";-import { getEvaluationConfigByDbConfigId } from "@/lib/platform/evaluation/config";+import { SERVER_EVENTS } from "@/constants/events";+import {+ authenticateOfflineApiKey,+ loadOfflineEvaluationConfig,+ EVALUATION_NOT_CONFIGURED_MESSAGE,+} from "@/lib/platform/evaluation/offline-auth";+import { normalizeThresholdScore } from "@/lib/platform/evaluation/threshold";+import {+ normalizeTypeConfig,+ persistEvaluationTypes,+ upsertEvaluationTypes,+ EvaluationTypeConfig,+} from "@/lib/platform/evaluation/type-config";import getMessage from "@/constants/messages";-import asaw from "@/utils/asaw";+import PostHogServer from "@/lib/posthog";import { errorResponse } from "@/helpers/server/response";+import { jsonParse } from "@/utils/json";export async function GET(request: Request) {- const authorizationHeader = request.headers.get("Authorization") || "";- if (!authorizationHeader.startsWith("Bearer ")) {- return errorResponse(getMessage().NO_API_KEY, 401);- }- const apiKey = authorizationHeader.replace(/^Bearer /, "").trim();- if (!apiKey) {- return errorResponse(getMessage().NO_API_KEY, 401);- }- const [keyErr, apiInfo] = await getAPIKeyInfo({ apiKey });- if (keyErr || !apiInfo?.databaseConfigId) {- return errorResponse(getMessage().NO_API_KEY, 401);- }+ const startTimestamp = Date.now();- const [configErr, config] = await asaw(- getEvaluationConfigByDbConfigId(apiInfo.databaseConfigId, true)+ const auth = await authenticateOfflineApiKey(request);+ if ("error" in auth) return auth.error;++ const loaded = await loadOfflineEvaluationConfig(auth.databaseConfigId, () =>+ errorResponse(EVALUATION_NOT_CONFIGURED_MESSAGE, 200, {+ eval_types: [],+ configured: false,+ }));- if (configErr || !config?.id) {- return errorResponse(- "Evaluation not configured. Set up evaluation in the OpenLIT dashboard first.",- 200,- { eval_types: [], configured: false }- );- }+ if ("error" in loaded) return loaded.error;- const types = ((config as any).evaluationTypes || []).map((t: any) => ({+ const types = (loaded.config.evaluationTypes || []).map((t: any) => ({id: t.id,label: t.label || t.id,description: t.description || "",enabled: !!t.enabled,is_custom: !!t.isCustom,+ ...(typeof t.thresholdScore === "number"+ ? { threshold_score: t.thresholdScore }+ : {}),}));+ PostHogServer.fireEvent({+ event: SERVER_EVENTS.EVALUATION_OFFLINE_TYPES_SUCCESS,+ startTimestamp,+ });return Response.json({ eval_types: types });}+export async function POST(request: Request) {+ const startTimestamp = Date.now();++ const auth = await authenticateOfflineApiKey(request);+ if ("error" in auth) return auth.error;++ let body: any;+ try {+ body = await request.json();+ } catch {+ return errorResponse("Invalid JSON body", 400);+ }+ const types = body.types as any[] | undefined;+ if (!Array.isArray(types)) {+ PostHogServer.fireEvent({+ event: SERVER_EVENTS.EVALUATION_OFFLINE_TYPE_CREATE_FAILURE,+ startTimestamp,+ });+ return errorResponse("Invalid types array", 400);+ }++ const thresholdScores: Array<number | undefined> = [];+ for (const t of types) {+ const normalized = normalizeThresholdScore(t?.thresholdScore);+ if (Number.isNaN(normalized)) {+ PostHogServer.fireEvent({+ event: SERVER_EVENTS.EVALUATION_OFFLINE_TYPE_CREATE_FAILURE,+ startTimestamp,+ });+ return errorResponse(getMessage().EVALUATION_THRESHOLD_SCORE_INVALID, 400);+ }+ thresholdScores.push(normalized);+ }++ const loaded = await loadOfflineEvaluationConfig(auth.databaseConfigId, () => {+ PostHogServer.fireEvent({+ event: SERVER_EVENTS.EVALUATION_OFFLINE_TYPE_CREATE_FAILURE,+ startTimestamp,+ });+ return errorResponse(EVALUATION_NOT_CONFIGURED_MESSAGE, 400);+ });+ if ("error" in loaded) return loaded.error;++ const normalizedTypes = types.map((t, i) =>+ normalizeTypeConfig(t, thresholdScores[i])+ );++ // Upsert into the existing meta list. Unlike the dashboard POST (which+ // always sends the full UI list), API-key clients may post a partial+ // array — a full replace would wipe unrelated types and thresholds.+ const meta = jsonParse((loaded.config as any).meta || "{}") as Record<+ string,+ any+ >;+ const existingTypes: EvaluationTypeConfig[] =+ (meta.evaluationTypes as EvaluationTypeConfig[]) || [];+ const nextTypes = upsertEvaluationTypes(existingTypes, normalizedTypes);++ await persistEvaluationTypes(+ loaded.config.id,+ (loaded.config as any).meta,+ nextTypes+ );++ PostHogServer.fireEvent({+ event: SERVER_EVENTS.EVALUATION_OFFLINE_TYPE_CREATE_SUCCESS,+ startTimestamp,+ });+ return Response.json({ data: normalizedTypes });+}+export async function OPTIONS() {return new Response(null, {status: 200,headers: {"Access-Control-Allow-Origin": "*",- "Access-Control-Allow-Methods": "GET, OPTIONS",+ "Access-Control-Allow-Methods": "GET, POST, OPTIONS","Access-Control-Allow-Headers": "Content-Type, Authorization",},});diff --git a/src/client/src/app/api/evaluation/types/[id]/route.ts b/src/client/src/app/api/evaluation/types/[id]/route.tsindex f10d5e98..8664af18 100644--- a/src/client/src/app/api/evaluation/types/[id]/route.ts+++ b/src/client/src/app/api/evaluation/types/[id]/route.ts@@ -1,38 +1,24 @@import { withAudit } from "@/lib/audit/route";import { withCurrentOrganisationPermission } from "@/lib/rbac/current";import { getEvaluationConfig } from "@/lib/platform/evaluation/config";-import { syncRuleEntitiesFromConfig } from "@/lib/platform/evaluation/sync-rule-entities";+import { normalizeThresholdScore } from "@/lib/platform/evaluation/threshold";+import {+ normalizeRules,+ mergeTypeIntoList,+ persistEvaluationTypes,+ EvaluationTypeConfig,+} from "@/lib/platform/evaluation/type-config";import { SERVER_EVENTS } from "@/constants/events";import PostHogServer from "@/lib/posthog";import { NextRequest } from "next/server";import asaw from "@/utils/asaw";-import { jsonParse, jsonStringify } from "@/utils/json";+import { jsonParse } from "@/utils/json";+import getMessage from "@/constants/messages";-export interface RuleWithPriority {- ruleId: string;- priority: number;-}--export interface EvaluationTypeConfig {- id: string;- enabled: boolean;- isCustom?: boolean;- label?: string;- description?: string;- rules?: RuleWithPriority[];- prompt?: string;- defaultPrompt?: string;-}--function normalizeRules(rules: any[]): RuleWithPriority[] {- if (!Array.isArray(rules)) return [];- return rules- .filter((r: any) => r?.ruleId)- .map((r: any) => ({- ruleId: r.ruleId,- priority: Number(r.priority) || 0,- }));-}+export type {+ RuleWithPriority,+ EvaluationTypeConfig,+} from "@/lib/platform/evaluation/type-config";async function GETHandler(_: NextRequest,@@ -63,7 +49,28 @@ async function PATCHHandler() {const startTimestamp = Date.now();const typeId = params.id;- const body = await request.json();+ let body: any;+ try {+ body = await request.json();+ } catch {+ return Response.json({ err: "Invalid JSON body" }, { status: 400 });+ }++ const thresholdScore =+ body.thresholdScore !== undefined+ ? normalizeThresholdScore(body.thresholdScore)+ : undefined;+ if (Number.isNaN(thresholdScore)) {+ PostHogServer.fireEvent({+ event: SERVER_EVENTS.EVALUATION_TYPE_UPDATE_FAILURE,+ startTimestamp,+ });+ return Response.json(+ { err: getMessage().EVALUATION_THRESHOLD_SCORE_INVALID },+ { status: 400 }+ );+ }+const [err, config] = await asaw(getEvaluationConfig(undefined, true, false));if (err || !config?.id) {PostHogServer.fireEvent({@@ -75,9 +82,8 @@ async function PATCHHandler({ status: 400 });}- const prisma = (await import("@/lib/prisma")).default;const meta = jsonParse((config as any).meta || "{}") as Record<string, any>;- let types: EvaluationTypeConfig[] =+ const types: EvaluationTypeConfig[] =(meta.evaluationTypes as EvaluationTypeConfig[]) || [];const idx = types.findIndex((t: any) => t.id === typeId);const existing = idx >= 0 ? types[idx] : { id: typeId, enabled: false, rules: [] };@@ -86,6 +92,10 @@ async function PATCHHandler(enabled: body.enabled ?? existing.enabled,rules: body.rules !== undefined ? normalizeRules(body.rules) : existing.rules || [],prompt: body.prompt !== undefined ? body.prompt : existing.prompt,+ thresholdScore:+ body.thresholdScore !== undefined+ ? thresholdScore+ : (existing as any).thresholdScore,};// Preserve or update custom type metadataif (body.isCustom || (existing as any).isCustom) {@@ -93,17 +103,13 @@ async function PATCHHandler(updated.label = body.label ?? (existing as any).label ?? typeId;updated.description = body.description ?? (existing as any).description ?? "";}- if (idx >= 0) {- types[idx] = updated;- } else {- types = [...types, updated];- }- meta.evaluationTypes = types;- await prisma.evaluationConfigs.update({- where: { id: config.id },- data: { meta: jsonStringify(meta) },- });- await syncRuleEntitiesFromConfig();++ await persistEvaluationTypes(+ config.id,+ (config as any).meta,+ mergeTypeIntoList(types, updated)+ );+PostHogServer.fireEvent({event: SERVER_EVENTS.EVALUATION_TYPE_UPDATE_SUCCESS,startTimestamp,@@ -128,7 +134,6 @@ async function DELETEHandler({ status: 400 });}- const prisma = (await import("@/lib/prisma")).default;const meta = jsonParse((config as any).meta || "{}") as Record<string, any>;const types: EvaluationTypeConfig[] =(meta.evaluationTypes as EvaluationTypeConfig[]) || [];@@ -145,12 +150,12 @@ async function DELETEHandler();}- meta.evaluationTypes = types.filter((t) => t.id !== typeId);- await prisma.evaluationConfigs.update({- where: { id: config.id },- data: { meta: jsonStringify(meta) },- });- await syncRuleEntitiesFromConfig();+ await persistEvaluationTypes(+ config.id,+ (config as any).meta,+ types.filter((t) => t.id !== typeId)+ );+PostHogServer.fireEvent({event: SERVER_EVENTS.EVALUATION_TYPE_DELETE_SUCCESS,startTimestamp,diff --git a/src/client/src/app/api/evaluation/types/route.ts b/src/client/src/app/api/evaluation/types/route.tsindex 19592e5a..0fba22d7 100644--- a/src/client/src/app/api/evaluation/types/route.ts+++ b/src/client/src/app/api/evaluation/types/route.ts@@ -1,56 +1,21 @@import { withAudit } from "@/lib/audit/route";import { withCurrentOrganisationPermission } from "@/lib/rbac/current";import { getEvaluationConfig } from "@/lib/platform/evaluation/config";-import { syncRuleEntitiesFromConfig } from "@/lib/platform/evaluation/sync-rule-entities";+import { normalizeThresholdScore } from "@/lib/platform/evaluation/threshold";+import {+ normalizeTypeConfig,+ persistEvaluationTypes,+} from "@/lib/platform/evaluation/type-config";import { SERVER_EVENTS } from "@/constants/events";import PostHogServer from "@/lib/posthog";import { NextRequest } from "next/server";import asaw from "@/utils/asaw";-import { jsonParse, jsonStringify } from "@/utils/json";+import getMessage from "@/constants/messages";-export interface RuleWithPriority {- ruleId: string;- priority: number;-}--export interface EvaluationTypeConfig {- id: string;- enabled: boolean;- isCustom?: boolean;- label?: string;- description?: string;- rules?: RuleWithPriority[];- ruleId?: string;- priority?: number;- defaultPrompt?: string;- prompt?: string;-}--function normalizeTypeConfig(t: any): EvaluationTypeConfig {- const rules = t.rules?.length- ? t.rules.filter((r: any) => r?.ruleId).map((r: any) => ({- ruleId: r.ruleId,- priority: Number(r.priority) || 0,- }))- : t.ruleId- ? [{ ruleId: t.ruleId, priority: Number(t.priority) || 0 }]- : [];- const config: EvaluationTypeConfig = {- id: t.id,- enabled: !!t.enabled,- rules,- };- // Preserve custom type metadata- if (t.isCustom) {- config.isCustom = true;- if (t.label) config.label = String(t.label).trim();- if (t.description) config.description = String(t.description).trim();- if (t.prompt) config.prompt = String(t.prompt).trim();- } else if (t.prompt) {- config.prompt = String(t.prompt).trim();- }- return config;-}+export type {+ RuleWithPriority,+ EvaluationTypeConfig,+} from "@/lib/platform/evaluation/type-config";async function GETHandler(_: NextRequest) {const [err, config] = await asaw(getEvaluationConfig(undefined, true, false));@@ -67,7 +32,12 @@ async function GETHandler(_: NextRequest) {async function POSTHandler(request: NextRequest) {const startTimestamp = Date.now();- const body = await request.json();+ let body: any;+ try {+ body = await request.json();+ } catch {+ return Response.json({ err: "Invalid JSON body" }, { status: 400 });+ }const types = body.types as any[] | undefined;if (!Array.isArray(types)) {PostHogServer.fireEvent({@@ -77,6 +47,22 @@ async function POSTHandler(request: NextRequest) {return Response.json({ err: "Invalid types array" }, { status: 400 });}+ const thresholdScores: Array<number | undefined> = [];+ for (const t of types) {+ const normalized = normalizeThresholdScore(t?.thresholdScore);+ if (Number.isNaN(normalized)) {+ PostHogServer.fireEvent({+ event: SERVER_EVENTS.EVALUATION_TYPE_CREATE_FAILURE,+ startTimestamp,+ });+ return Response.json(+ { err: getMessage().EVALUATION_THRESHOLD_SCORE_INVALID },+ { status: 400 }+ );+ }+ thresholdScores.push(normalized);+ }+const [err, config] = await asaw(getEvaluationConfig(undefined, true, false));if (err || !config?.id) {PostHogServer.fireEvent({@@ -86,18 +72,11 @@ async function POSTHandler(request: NextRequest) {return Response.json({ err: "Evaluation config not found" }, { status: 400 });}- const normalizedTypes = types.map(normalizeTypeConfig);-- const prisma = (await import("@/lib/prisma")).default;- const meta = jsonParse((config as any).meta || "{}") as Record<string, any>;- meta.evaluationTypes = normalizedTypes;-- await prisma.evaluationConfigs.update({- where: { id: config.id },- data: { meta: jsonStringify(meta) },- });+ const normalizedTypes = types.map((t, i) =>+ normalizeTypeConfig(t, thresholdScores[i])+ );- await syncRuleEntitiesFromConfig();+ await persistEvaluationTypes(config.id, (config as any).meta, normalizedTypes);PostHogServer.fireEvent({event: SERVER_EVENTS.EVALUATION_TYPE_CREATE_SUCCESS,diff --git a/src/client/src/constants/events.ts b/src/client/src/constants/events.tsindex 80150293..c8f056fc 100644--- a/src/client/src/constants/events.ts+++ b/src/client/src/constants/events.ts@@ -259,6 +259,12 @@ export const SERVER_EVENTS = {EVALUATION_LLM_RUN_FAILURE: "EVALUATION_LLM_RUN_FAILURE",EVALUATION_OFFLINE_SUCCESS: "EVALUATION_OFFLINE_SUCCESS",EVALUATION_OFFLINE_FAILURE: "EVALUATION_OFFLINE_FAILURE",+ EVALUATION_OFFLINE_TYPES_SUCCESS: "EVALUATION_OFFLINE_TYPES_SUCCESS",+ EVALUATION_OFFLINE_TYPES_FAILURE: "EVALUATION_OFFLINE_TYPES_FAILURE",+ EVALUATION_OFFLINE_TYPE_CREATE_SUCCESS: "EVALUATION_OFFLINE_TYPE_CREATE_SUCCESS",+ EVALUATION_OFFLINE_TYPE_CREATE_FAILURE: "EVALUATION_OFFLINE_TYPE_CREATE_FAILURE",+ EVALUATION_OFFLINE_TYPE_UPDATE_SUCCESS: "EVALUATION_OFFLINE_TYPE_UPDATE_SUCCESS",+ EVALUATION_OFFLINE_TYPE_UPDATE_FAILURE: "EVALUATION_OFFLINE_TYPE_UPDATE_FAILURE",// UserUSER_ONBOARDING_COMPLETE_SUCCESS: "USER_ONBOARDING_COMPLETE_SUCCESS",diff --git a/src/client/src/constants/messages/en.ts b/src/client/src/constants/messages/en.tsindex 91fef011..5dd2b0bf 100644--- a/src/client/src/constants/messages/en.ts+++ b/src/client/src/constants/messages/en.ts@@ -115,6 +115,12 @@ export const EVALUATION_SAMPLE_RATE_DESCRIPTION ="Percentage of traces to auto-evaluate. Uses deterministic sampling per trace to reduce LLM cost. Traces not selected are marked as skipped and are not re-queried on future cron runs; manual evaluation is always available.";export const EVALUATION_SAMPLE_RATE_INVALID ="Sample rate must be a number between 0 and 1.";+export const EVALUATION_THRESHOLD_SCORE_INVALID =+ "Threshold score must be a number between 0 and 1.";+export const EVALUATION_TYPE_THRESHOLD_LABEL = "Verdict threshold";+export const EVALUATION_TYPE_THRESHOLD_DESCRIPTION =+ "Score above which this type's verdict is flagged as \"yes\". Applies to dashboard, auto, and offline evaluations. Leave blank to use the default threshold (0.5).";+export const EVALUATION_TYPE_THRESHOLD_PLACEHOLDER = "0.5 (default)";export const EVALUATION_SAMPLE_RATE_PERCENT_INVALID ="Sample rate must be a number between 0 and 100.";export const EVALUATION_SAVING = "Saving...";diff --git a/src/client/src/lib/platform/evaluation/config.ts b/src/client/src/lib/platform/evaluation/config.tsindex ccf99f78..c3550798 100644--- a/src/client/src/lib/platform/evaluation/config.ts+++ b/src/client/src/lib/platform/evaluation/config.ts@@ -47,6 +47,7 @@ export interface EvaluationTypeWithPrompt {rules?: Array<{ ruleId: string; priority: number }>;prompt?: string;defaultPrompt: string;+ thresholdScore?: number;}async function buildEvaluationTypesWithPrompts(@@ -61,6 +62,7 @@ async function buildEvaluationTypesWithPrompts(isCustom?: boolean;rules?: Array<{ ruleId: string; priority: number }>;prompt?: string;+ thresholdScore?: number;}>) || [];const overrideMap = new Map(@@ -82,6 +84,10 @@ async function buildEvaluationTypesWithPrompts(rules: override?.rules?.filter((r) => r?.ruleId) ?? [],prompt: override?.prompt,defaultPrompt: defaultPrompts[et.id] ?? "",+ thresholdScore:+ typeof override?.thresholdScore === "number"+ ? override.thresholdScore+ : undefined,};});@@ -98,6 +104,8 @@ async function buildEvaluationTypesWithPrompts(rules: t.rules?.filter((r) => r?.ruleId) ?? [],prompt: t.prompt,defaultPrompt: "",+ thresholdScore:+ typeof t.thresholdScore === "number" ? t.thresholdScore : undefined,}));return [...builtInTypes, ...customTypes];diff --git a/src/client/src/lib/platform/evaluation/index.ts b/src/client/src/lib/platform/evaluation/index.tsindex 71290c5a..7a2aebfb 100644--- a/src/client/src/lib/platform/evaluation/index.ts+++ b/src/client/src/lib/platform/evaluation/index.ts@@ -7,7 +7,6 @@ import { OPENLIT_EVALUATION_TABLE_NAME } from "./table-details";import { runEvaluation } from "./run-evaluation";import { getEvaluationConfig, getEvaluationConfigById } from "./config";import asaw from "@/utils/asaw";-import { evaluateRules } from "@/lib/platform/rule-engine/evaluate";import {AutoEvaluationConfig,Evaluation,@@ -21,6 +20,8 @@ import { getTraceMappingKeyFullPath } from "@/helpers/server/trace";import {getContextFromRuleEngineForTrace,getContextFromRulesWithPriority,+ getContextFromFields,+ getContextFromRulesWithPriorityForFields,} from "./rule-engine-context";import { TraceRow } from "@/types/trace";import { get } from "lodash";@@ -50,6 +51,56 @@ import {type EvaluationTypeRef,} from "@/helpers/client/evaluation-type";+/**+ * Collects rule links (with priority) across a set of enabled evaluation+ * types. Shared by the real-time/auto and offline evaluation paths so both+ * resolve rule-engine context identically for the same configured types.+ */+function collectRulesWithPriority(+ types: Array<{+ rules?: Array<{ ruleId: string; priority: number }>;+ ruleId?: string;+ priority?: number;+ }>+): Array<{ ruleId: string; priority: number }> {+ const rulesWithPriority: Array<{ ruleId: string; priority: number }> = [];+ for (const t of types) {+ if (t.rules?.length) {+ rulesWithPriority.push(+ ...t.rules.filter((r) => r.ruleId && r.ruleId.trim())+ );+ } else if (t.ruleId) {+ rulesWithPriority.push({+ ruleId: t.ruleId,+ priority: t.priority ?? 0,+ });+ }+ }+ return rulesWithPriority;+}++function applyPerTypeThresholds(+ results: Evaluation[],+ enabledTypes: Array<{ id: string; label?: string; thresholdScore?: number }>,+ fallbackThreshold: number+): Evaluation[] {+ return results.map((evaluation) => {+ const type = enabledTypes.find(+ (t) =>+ t.id.toLowerCase() === evaluation.evaluation.toLowerCase() ||+ t.label?.toLowerCase() === evaluation.evaluation.toLowerCase()+ );+ const threshold =+ typeof type?.thresholdScore === "number"+ ? type.thresholdScore+ : fallbackThreshold;+ return {+ ...evaluation,+ verdict: evaluation.score > threshold ? "yes" : "no",+ };+ });+}+/*** Cost from Manage Models (`openlit_provider_models`) — same source as* Pricing / Openground. Returns 0 if the model is not configured.@@ -475,15 +526,18 @@ async function getEvaluationConfigForTrace(const prompt = get(trace, getTraceMappingKeyFullPath("prompt", true));const dbConfig = dbConfigId ?? evaluationConfig.databaseConfigId;+ const DEFAULT_THRESHOLD = 0.5;const evaluationTypes =((evaluationConfig as any).evaluationTypes || []) as Array<{id: string;enabled: boolean;+ label?: string;rules?: Array<{ ruleId: string; priority: number }>;ruleId?: string;priority?: number;prompt?: string;defaultPrompt?: string;+ thresholdScore?: number;}>;// Default: Hallucination, Bias, Toxicity enabled@@ -498,19 +552,7 @@ async function getEvaluationConfigForTrace().map((t) => ({ ...t, enabled: true }));// Collect rules with priority from enabled types- const rulesWithPriority: Array<{ ruleId: string; priority: number }> = [];- for (const t of enabledTypes) {- if (t.rules?.length) {- rulesWithPriority.push(- ...t.rules.filter((r) => r.ruleId && r.ruleId.trim())- );- } else if (t.ruleId) {- rulesWithPriority.push({- ruleId: t.ruleId,- priority: t.priority ?? 0,- });- }- }+ const rulesWithPriority = collectRulesWithPriority(enabledTypes);let contextContents: string[];let matchingRuleIds: string[];@@ -553,7 +595,7 @@ async function getEvaluationConfigForTrace(prompt: prompt ?? "",contexts: finalContextString,response: response ?? "",- thresholdScore: 0.5,+ thresholdScore: DEFAULT_THRESHOLD,});@@ -561,6 +603,14 @@ async function getEvaluationConfigForTrace(return { success: false, error: data.error };}+ // Recompute verdicts with per-type thresholds, matching the offline+ // path so dashboard/auto/manual and SDK runs honor the same config.+ const evaluations = applyPerTypeThresholds(+ data.result || [],+ enabledTypes,+ DEFAULT_THRESHOLD+ );+const metaBase: Record<string, string> = {model: `${evaluationConfig.provider}/${evaluationConfig.model}`,traceTimeStamp: String(trace.Timestamp ?? ""),@@ -584,7 +634,7 @@ async function getEvaluationConfigForTrace(const storeResult = await storeEvaluation(trace.SpanId,- data.result || [],+ evaluations,metaBase,dbConfig,evaluationTypes.map((t) => ({ id: t.id, label: (t as any).label || t.id }))@@ -867,17 +917,10 @@ export async function runOfflineEvaluation(priority?: number;prompt?: string;defaultPrompt?: string;+ thresholdScore?: number;}>;- let enabledTypes = requestedTypes?.length- ? allTypes.filter((t) => requestedTypes.includes(t.id))- : allTypes.filter((t) => t.enabled);-- if (enabledTypes.length === 0) {- enabledTypes = allTypes- .filter((t) => ["hallucination", "bias", "toxicity"].includes(t.id))- .map((t) => ({ ...t, enabled: true }));- }+ let enabledTypes: typeof allTypes;if (requestedTypes?.length) {const allTypeIds = new Set(allTypes.map((t) => t.id));@@ -888,39 +931,50 @@ export async function runOfflineEvaluation(error: `Unknown eval types: ${unknown.join(", ")}`,};}++ const disabled = requestedTypes.filter((id) => {+ const type = allTypes.find((t) => t.id === id);+ return type && !type.enabled;+ });+ if (disabled.length > 0) {+ return {+ success: false,+ error: `Disabled eval types: ${disabled.join(", ")}`,+ };+ }++ enabledTypes = allTypes.filter((t) => requestedTypes.includes(t.id));+ } else {+ enabledTypes = allTypes.filter((t) => t.enabled);++ if (enabledTypes.length === 0) {+ enabledTypes = allTypes+ .filter((t) => ["hallucination", "bias", "toxicity"].includes(t.id))+ .map((t) => ({ ...t, enabled: true }));+ }}+ // Collect rules with priority from enabled types, same as the real-time/+ // auto path, so offline evals resolve rule-engine context identically for+ // the same configured evaluation types.+ const rulesWithPriority = collectRulesWithPriority(enabledTypes);+let contextContents: string[] = [];let matchingRuleIds: string[] = [];let contextEntityIds: string[] = [];if (attributes && Object.keys(attributes).length > 0) {- try {- const ruleResult = await evaluateRules(- {- fields: attributes,- entity_type: "context" as any,- include_entity_data: true,- },- databaseConfigId- );-- matchingRuleIds = ruleResult.matchingRuleIds || [];-- if (ruleResult.entity_data) {- for (const [key, entityData] of Object.entries(- ruleResult.entity_data- )) {- if (entityData?.content) {- contextContents.push(String(entityData.content));- const match = key.match(/^context:(.+)$/);- if (match) contextEntityIds.push(match[1]);- }- }- }- } catch (e) {- consoleLog("Rule engine evaluation failed for offline eval:", e);- }+ const result =+ rulesWithPriority.length > 0+ ? await getContextFromRulesWithPriorityForFields(+ attributes,+ rulesWithPriority,+ databaseConfigId+ )+ : await getContextFromFields(attributes, databaseConfigId);+ contextContents = result.contextContents;+ matchingRuleIds = result.matchingRuleIds;+ contextEntityIds = result.contextEntityIds || [];}const userContextsCount = userContexts?.length || 0;@@ -952,6 +1006,12 @@ export async function runOfflineEvaluation(return { success: false, error: data.error };}+ const evaluations = applyPerTypeThresholds(+ data.result || [],+ enabledTypes,+ thresholdScore+ );+const metaBase: Record<string, string> = {model: `${evaluationConfig.provider}/${evaluationConfig.model}`,ruleIds: matchingRuleIds.join(","),@@ -988,7 +1048,7 @@ export async function runOfflineEvaluation(}));const storeResult = await storeEvaluation(spanId,- data.result || [],+ evaluations,metaBase,databaseConfigId,customTypes@@ -1000,7 +1060,7 @@ export async function runOfflineEvaluation(return {success: true,- evaluations: data.result || [],+ evaluations,contextApplied: {ruleMatched: matchingRuleIds.length > 0,matchingRuleIds,diff --git a/src/client/src/lib/platform/evaluation/offline-auth.ts b/src/client/src/lib/platform/evaluation/offline-auth.tsnew file mode 100644index 00000000..5409eb80--- /dev/null+++ b/src/client/src/lib/platform/evaluation/offline-auth.ts@@ -0,0 +1,56 @@+import { getAPIKeyInfo } from "@/lib/platform/api-keys";+import { getEvaluationConfigByDbConfigId } from "./config";+import getMessage from "@/constants/messages";+import asaw from "@/utils/asaw";+import { errorResponse } from "@/helpers/server/response";+import { EvaluationConfigWithSecret } from "@/types/evaluation";++export type OfflineEvaluationTypeConfig = EvaluationConfigWithSecret & {+ evaluationTypes?: any[];+};++/**+ * Authenticates an offline/SDK request via its Bearer API key. Shared by+ * every /api/evaluation/offline/types handler so the auth check and its+ * 401 responses can't drift between them.+ */+export async function authenticateOfflineApiKey(+ request: Request+): Promise<{ error: Response } | { databaseConfigId: string }> {+ const authorizationHeader = request.headers.get("Authorization") || "";+ if (!authorizationHeader.startsWith("Bearer ")) {+ return { error: errorResponse(getMessage().NO_API_KEY, 401) };+ }+ const apiKey = authorizationHeader.replace(/^Bearer /, "").trim();+ if (!apiKey) {+ return { error: errorResponse(getMessage().NO_API_KEY, 401) };+ }+ const [keyErr, apiInfo] = await getAPIKeyInfo({ apiKey });+ if (keyErr || !apiInfo?.databaseConfigId) {+ return { error: errorResponse(getMessage().NO_API_KEY, 401) };+ }+ return { databaseConfigId: apiInfo.databaseConfigId };+}++/**+ * Loads the evaluation config for an authenticated offline/SDK request's+ * own database config, scoped by databaseConfigId. `notConfiguredResponse`+ * lets each caller shape its own "not configured" response (e.g. GET+ * degrades gracefully with a 200, while mutating routes return a 400) while+ * still sharing the lookup and error-detection logic.+ */+export async function loadOfflineEvaluationConfig(+ databaseConfigId: string,+ notConfiguredResponse: () => Response+): Promise<{ error: Response } | { config: OfflineEvaluationTypeConfig }> {+ const [configErr, config] = await asaw(+ getEvaluationConfigByDbConfigId(databaseConfigId, true)+ );+ if (configErr || !config?.id) {+ return { error: notConfiguredResponse() };+ }+ return { config: config as OfflineEvaluationTypeConfig };+}++export const EVALUATION_NOT_CONFIGURED_MESSAGE =+ "Evaluation not configured. Set up evaluation in the OpenLIT dashboard first.";diff --git a/src/client/src/lib/platform/evaluation/rule-engine-context.ts b/src/client/src/lib/platform/evaluation/rule-engine-context.tsindex 57162e70..6afa903a 100644--- a/src/client/src/lib/platform/evaluation/rule-engine-context.ts+++ b/src/client/src/lib/platform/evaluation/rule-engine-context.ts@@ -65,17 +65,20 @@ export interface RuleEngineContextResult {}/**- * Fetches context content from the rule engine for a given trace.- * Evaluates rules using trace attributes and returns concatenated context- * content from matching context entities.+ * Fetches context content from the rule engine for a set of rule-engine+ * input fields. Evaluates rules against those fields and returns+ * concatenated context content from matching context entities.+ *+ * This is the shared primitive behind both the trace-based (real-time/auto)+ * and fields-based (offline/SDK) evaluation paths, so both resolve context+ * identically given equivalent input.** @returns Context contents and matching rule IDs (for display and storage)*/-export async function getContextFromRuleEngineForTrace(- trace: TraceRow,+export async function getContextFromFields(+ fields: Record<string, string | number | boolean>,databaseConfigId?: string): Promise<RuleEngineContextResult> {- const fields = extractRuleEngineFieldsFromTrace(trace);if (Object.keys(fields).length === 0) {return { contextContents: [], matchingRuleIds: [], contextEntityIds: [] };}@@ -117,21 +120,39 @@ export async function getContextFromRuleEngineForTrace(}}+/**+ * Fetches context content from the rule engine for a given trace.+ * Evaluates rules using trace attributes and returns concatenated context+ * content from matching context entities.+ *+ * @returns Context contents and matching rule IDs (for display and storage)+ */+export async function getContextFromRuleEngineForTrace(+ trace: TraceRow,+ databaseConfigId?: string+): Promise<RuleEngineContextResult> {+ return getContextFromFields(+ extractRuleEngineFieldsFromTrace(trace),+ databaseConfigId+ );+}+export interface RuleWithPriority {ruleId: string;priority: number;}/**- * Fetches context from rules in priority order. Used when evaluation types- * config specifies which rules to use for auto evaluation.+ * Fetches context from rules in priority order, given rule-engine input+ * fields directly. Shared primitive behind both the trace-based+ * (real-time/auto) and fields-based (offline/SDK) evaluation paths.*/-export async function getContextFromRulesWithPriority(- trace: TraceRow,+export async function getContextFromRulesWithPriorityForFields(+ fields: Record<string, string | number | boolean>,rulesWithPriority: RuleWithPriority[],databaseConfigId?: string): Promise<RuleEngineContextResult> {- const result = await getContextFromRuleEngineForTrace(trace, databaseConfigId);+ const result = await getContextFromFields(fields, databaseConfigId);if (rulesWithPriority.length === 0) return result;const priorityByRule = new Map(@@ -148,7 +169,6 @@ export async function getContextFromRulesWithPriority();// Re-fetch with entity data to get ordered context- const fields = extractRuleEngineFieldsFromTrace(trace);if (Object.keys(fields).length === 0) {return {contextContents: [],@@ -194,3 +214,19 @@ export async function getContextFromRulesWithPriority(return result;}}++/**+ * Fetches context from rules in priority order. Used when evaluation types+ * config specifies which rules to use for auto evaluation.+ */+export async function getContextFromRulesWithPriority(+ trace: TraceRow,+ rulesWithPriority: RuleWithPriority[],+ databaseConfigId?: string+): Promise<RuleEngineContextResult> {+ return getContextFromRulesWithPriorityForFields(+ extractRuleEngineFieldsFromTrace(trace),+ rulesWithPriority,+ databaseConfigId+ );+}diff --git a/src/client/src/lib/platform/evaluation/threshold.ts b/src/client/src/lib/platform/evaluation/threshold.tsnew file mode 100644index 00000000..e8f6f69c--- /dev/null+++ b/src/client/src/lib/platform/evaluation/threshold.ts@@ -0,0 +1,21 @@+/**+ * Validates and clamps a per-type threshold override. Returns undefined when+ * no value was provided (caller should fall back to the request/global+ * default), or NaN when a value was provided but isn't a usable number+ * (caller should reject the write).+ */+export function normalizeThresholdScore(value: unknown): number | undefined {+ if (value === undefined || value === null || value === "") {+ return undefined;+ }++ const numeric =+ typeof value === "number" ? value : Number.parseFloat(String(value));++ if (!Number.isFinite(numeric)) {+ return Number.NaN;+ }+ if (numeric < 0) return 0;+ if (numeric > 1) return 1;+ return numeric;+}diff --git a/src/client/src/lib/platform/evaluation/type-config.ts b/src/client/src/lib/platform/evaluation/type-config.tsnew file mode 100644index 00000000..b4ac27f3--- /dev/null+++ b/src/client/src/lib/platform/evaluation/type-config.ts@@ -0,0 +1,123 @@+import { jsonParse, jsonStringify } from "@/utils/json";+import { syncRuleEntitiesFromConfig } from "./sync-rule-entities";++export interface RuleWithPriority {+ ruleId: string;+ priority: number;+}++export interface EvaluationTypeConfig {+ id: string;+ enabled: boolean;+ isCustom?: boolean;+ label?: string;+ description?: string;+ rules?: RuleWithPriority[];+ ruleId?: string;+ priority?: number;+ defaultPrompt?: string;+ prompt?: string;+ thresholdScore?: number;+}++/**+ * Normalizes a raw rules array (as received from a request body) into+ * validated { ruleId, priority } pairs, dropping entries without a ruleId.+ */+export function normalizeRules(rules: any[]): RuleWithPriority[] {+ if (!Array.isArray(rules)) return [];+ return rules+ .filter((r: any) => r?.ruleId)+ .map((r: any) => ({+ ruleId: r.ruleId,+ priority: Number(r.priority) || 0,+ }));+}++/**+ * Normalizes a single raw evaluation type entry (as received from a request+ * body) into a persistable EvaluationTypeConfig. The threshold is validated+ * and normalized by the caller (via normalizeThresholdScore) beforehand,+ * since only the caller knows how to surface a validation error.+ */+export function normalizeTypeConfig(+ t: any,+ thresholdScore: number | undefined+): EvaluationTypeConfig {+ const rules = t.rules?.length+ ? normalizeRules(t.rules)+ : t.ruleId+ ? [{ ruleId: t.ruleId, priority: Number(t.priority) || 0 }]+ : [];+ const config: EvaluationTypeConfig = {+ id: t.id,+ enabled: !!t.enabled,+ rules,+ };+ if (thresholdScore !== undefined) config.thresholdScore = thresholdScore;+ // Preserve custom type metadata+ if (t.isCustom) {+ config.isCustom = true;+ if (t.label) config.label = String(t.label).trim();+ if (t.description) config.description = String(t.description).trim();+ if (t.prompt) config.prompt = String(t.prompt).trim();+ } else if (t.prompt) {+ config.prompt = String(t.prompt).trim();+ }+ return config;+}++/**+ * Upserts a single type into an existing evaluationTypes list by id,+ * appending it when no entry with that id exists yet.+ */+export function mergeTypeIntoList(+ types: EvaluationTypeConfig[],+ updated: EvaluationTypeConfig+): EvaluationTypeConfig[] {+ const idx = types.findIndex((t) => t.id === updated.id);+ if (idx >= 0) {+ const next = [...types];+ next[idx] = updated;+ return next;+ }+ return [...types, updated];+}++/**+ * Upserts a batch of types into an existing list by id. Used by the+ * API-key offline create endpoint so posting a partial list cannot wipe+ * unrelated types (and their thresholdScore / rules) already in meta.+ */+export function upsertEvaluationTypes(+ existing: EvaluationTypeConfig[],+ incoming: EvaluationTypeConfig[]+): EvaluationTypeConfig[] {+ return incoming.reduce(+ (acc, type) => mergeTypeIntoList(acc, type),+ existing+ );+}++/**+ * Persists an evaluationTypes array into an evaluation config's meta JSON+ * and re-syncs rule-engine entities. Shared by every route (dashboard and+ * API-key-authed) that creates or updates evaluation types, so the+ * persistence step behaves identically regardless of caller.+ */+export async function persistEvaluationTypes(+ configId: string,+ configMeta: string | null | undefined,+ evaluationTypes: EvaluationTypeConfig[]+): Promise<void> {+ const prisma = (await import("@/lib/prisma")).default;+ const meta = jsonParse(configMeta || "{}") as Record<string, any>;+ meta.evaluationTypes = evaluationTypes;++ await prisma.evaluationConfigs.update({+ where: { id: configId },+ data: { meta: jsonStringify(meta) },+ });++ await syncRuleEntitiesFromConfig();+}