Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

feat: rule engine SDK, custom evaluation types, 11 built-in evals, and UI fixes (#1076)

Aman Agarwal5bba0cd47f20
diff --git a/docs/latest/openlit/evaluations/overview.mdx b/docs/latest/openlit/evaluations/overview.mdx
index c3e15c09..2895cf13 100644
--- a/docs/latest/openlit/evaluations/overview.mdx
+++ b/docs/latest/openlit/evaluations/overview.mdx
@@ -47,6 +47,10 @@ OpenLIT includes 11 evaluation types out of the box. Each can be independently e
**Context is the source of truth**: When context is provided (via the Rule Engine), evaluations judge the LLM response against the provided context — not against real-world knowledge. This enables domain-specific evaluation where your custom knowledge is authoritative.
</Info>
+### Custom evaluation types
+
+Beyond the 11 built-in types, you can create custom evaluation types with your own prompts. Navigate to **Evaluations > Settings > Evaluation Types** and click **Create Custom Type**. Each custom type requires an id (lowercase with underscores), a label, a description, and an evaluation prompt. Custom types run alongside built-in types in both auto and manual evaluations, and can be linked to Rule Engine rules just like built-in types. Unlike built-in types, custom types can be deleted when no longer needed.
+
## AI evaluation methods
OpenLIT provides automated LLM evaluation and testing capabilities for production AI applications:
diff --git a/docs/latest/sdk/features/rule-engine.mdx b/docs/latest/sdk/features/rule-engine.mdx
index 881ab02a..0c062bd7 100644
--- a/docs/latest/sdk/features/rule-engine.mdx
+++ b/docs/latest/sdk/features/rule-engine.mdx
@@ -221,7 +221,7 @@ Fetch compiled prompts from the Prompt Hub with variable substitution.
### Check Evaluation Rules
-Determine which evaluation types (hallucination, bias, etc.) are linked to rules matching the current trace.
+Determine which evaluation types (hallucination, bias, etc.) are linked to rules matching the current trace. This works with both the 11 built-in evaluation types and any custom evaluation types you have created.
<Tabs>
<Tab title="Python">
diff --git a/docs/snippets/llm-as-a-judge.mdx b/docs/snippets/llm-as-a-judge.mdx
index 644942b7..d403c6a5 100644
--- a/docs/snippets/llm-as-a-judge.mdx
+++ b/docs/snippets/llm-as-a-judge.mdx
@@ -96,6 +96,17 @@ Click on any evaluation type to configure its custom prompt and linked rules:
<img src="/images/evaluation-types-settings-detail.png" />
</Frame>
+### Custom evaluation types
+
+In addition to the 11 built-in types, you can create your own evaluation types tailored to your use case. From the **Evaluation Types** tab, click **Create Custom Type** and provide:
+
+- **ID**: A unique identifier in `lowercase_underscore` format (e.g., `domain_accuracy`)
+- **Label**: A human-readable name displayed in the UI
+- **Description**: A brief explanation of what this evaluation checks
+- **Evaluation Prompt**: The prompt used by the LLM judge. It should start with a `[Label evaluation context]` header describing the evaluation criteria
+
+Custom types work exactly like built-in types — they can be enabled or disabled, linked to Rule Engine rules, and run in both auto and manual evaluations. The only difference is that custom types can be deleted when they are no longer needed, while built-in types cannot.
+
### Run from a trace
<Frame>
diff --git a/src/client/package-lock.json b/src/client/package-lock.json
index 1438bf5c..7b6ad745 100644
--- a/src/client/package-lock.json
+++ b/src/client/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "openlit",
- "version": "1.17.0",
+ "version": "1.17.1",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "openlit",
- "version": "1.17.0",
+ "version": "1.17.1",
"dependencies": {
"@ai-sdk/anthropic": "^3.0.6",
"@ai-sdk/cohere": "^3.0.3",
diff --git a/src/client/package.json b/src/client/package.json
index da8aa8e5..b4c98c64 100644
--- a/src/client/package.json
+++ b/src/client/package.json
@@ -1,6 +1,6 @@
{
"name": "openlit",
- "version": "1.17.0",
+ "version": "1.17.1",
"private": true,
"scripts": {
"dev": "next dev",
diff --git a/src/client/src/__tests__/constants/messages-hi.test.ts b/src/client/src/__tests__/constants/messages-hi.test.ts
new file mode 100644
index 00000000..2517b467
--- /dev/null
+++ b/src/client/src/__tests__/constants/messages-hi.test.ts
@@ -0,0 +1,40 @@
+import * as hi from "@/constants/messages/hi";
+import * as en from "@/constants/messages/en";
+
+describe("Hindi messages (hi.ts)", () => {
+ const hiExports = Object.keys(hi);
+ const enExports = Object.keys(en);
+
+ it("exports at least one message", () => {
+ expect(hiExports.length).toBeGreaterThan(0);
+ });
+
+ it("all exported values are strings or arrays", () => {
+ for (const key of hiExports) {
+ const val = (hi as any)[key];
+ expect(typeof val === "string" || Array.isArray(val)).toBe(true);
+ }
+ });
+
+ it("no empty string values", () => {
+ for (const key of hiExports) {
+ const val = (hi as any)[key];
+ if (typeof val === "string") {
+ expect(val.length).toBeGreaterThan(0);
+ }
+ }
+ });
+
+ it("covers core message keys from en.ts", () => {
+ const coreKeys = [
+ "DATABASE_CONFIG_NOT_FOUND",
+ "UNAUTHORIZED_USER",
+ "MALFORMED_INPUTS",
+ "OPERATION_FAILED",
+ "NO_API_KEY",
+ ];
+ for (const key of coreKeys) {
+ expect(hiExports).toContain(key);
+ }
+ });
+});
diff --git a/src/client/src/__tests__/lib/platform/evaluation/evaluation-config.test.ts b/src/client/src/__tests__/lib/platform/evaluation/evaluation-config.test.ts
index dc91abf8..667841f4 100644
--- a/src/client/src/__tests__/lib/platform/evaluation/evaluation-config.test.ts
+++ b/src/client/src/__tests__/lib/platform/evaluation/evaluation-config.test.ts
@@ -21,6 +21,7 @@ jest.mock('@/lib/prisma', () => ({
default: {
evaluationConfigs: {
findFirst: jest.fn(),
+ findMany: jest.fn(),
update: jest.fn(),
create: jest.fn(),
},
@@ -62,7 +63,7 @@ jest.mock('path', () => ({
dirname: jest.fn((p: string) => p.split('/').slice(0, -1).join('/')),
}));
-import { getEvaluationConfig, setEvaluationConfig, getEvaluationConfigById } from '@/lib/platform/evaluation/config';
+import { getEvaluationConfig, setEvaluationConfig, getEvaluationConfigById, restoreEvaluationCronJobs } from '@/lib/platform/evaluation/config';
import { getEvaluationTypeDefaultPrompts } from '@/lib/platform/evaluation/evaluation-type-defaults';
import { getDBConfigByUser } from '@/lib/db-config';
import prisma from '@/lib/prisma';
@@ -398,3 +399,447 @@ describe('getEvaluationConfigById', () => {
expect(result.secret).toEqual({});
});
});
+
+// ---------------------------------------------------------------------------
+// buildEvaluationTypesWithPrompts (tested via getEvaluationConfig / getEvaluationConfigById)
+// ---------------------------------------------------------------------------
+
+describe('buildEvaluationTypesWithPrompts — custom evaluation types', () => {
+ /**
+ * Helper: create an eval config whose meta JSON contains the given evaluationTypes array.
+ */
+ const makeConfigWithMeta = (evaluationTypes: Array<Record<string, any>>) => ({
+ ...makeMockEvalConfig(),
+ meta: JSON.stringify({ evaluationTypes }),
+ });
+
+ it('includes all 11 built-in types when meta has no evaluationTypes', async () => {
+ (asaw as jest.Mock).mockResolvedValueOnce([null, makeMockEvalConfig()]);
+
+ const result = await getEvaluationConfigById('eval-cfg-1');
+
+ expect(result.evaluationTypes).toBeDefined();
+ expect(result.evaluationTypes!.length).toBe(11);
+ result.evaluationTypes!.forEach((t) => {
+ expect(t.isCustom).toBe(false);
+ });
+ });
+
+ it('built-in types use enabledByDefault when no override is present', async () => {
+ (asaw as jest.Mock).mockResolvedValueOnce([null, makeMockEvalConfig()]);
+
+ const result = await getEvaluationConfigById('eval-cfg-1');
+
+ const hallucination = result.evaluationTypes!.find((t) => t.id === 'hallucination');
+ expect(hallucination).toBeDefined();
+ expect(hallucination!.enabled).toBe(true); // enabledByDefault = true
+ expect(hallucination!.isCustom).toBe(false);
+
+ const safety = result.evaluationTypes!.find((t) => t.id === 'safety');
+ expect(safety).toBeDefined();
+ expect(safety!.enabled).toBe(false); // enabledByDefault = false
+ });
+
+ it('custom types in meta.evaluationTypes that are not in EVALUATION_TYPES get included with isCustom: true', async () => {
+ const config = makeConfigWithMeta([
+ {
+ id: 'my_custom_eval',
+ enabled: true,
+ label: 'My Custom Eval',
+ description: 'A custom evaluation type',
+ prompt: 'Check for custom issues',
+ },
+ ]);
+ (asaw as jest.Mock).mockResolvedValueOnce([null, config]);
+
+ const result = await getEvaluationConfigById('eval-cfg-1');
+
+ // Should have 11 built-in + 1 custom = 12
+ expect(result.evaluationTypes!.length).toBe(12);
+
+ const custom = result.evaluationTypes!.find((t) => t.id === 'my_custom_eval');
+ expect(custom).toBeDefined();
+ expect(custom!.isCustom).toBe(true);
+ expect(custom!.label).toBe('My Custom Eval');
+ expect(custom!.description).toBe('A custom evaluation type');
+ expect(custom!.prompt).toBe('Check for custom issues');
+ expect(custom!.enabled).toBe(true);
+ expect(custom!.enabledByDefault).toBe(false);
+ expect(custom!.defaultPrompt).toBe('');
+ });
+
+ it('custom types preserve label, description, and prompt from meta', async () => {
+ const config = makeConfigWithMeta([
+ {
+ id: 'brand_voice',
+ enabled: true,
+ label: 'Brand Voice',
+ description: 'Checks if the response matches our brand tone',
+ prompt: 'Evaluate whether the response uses our brand voice consistently',
+ },
+ ]);
+ (asaw as jest.Mock).mockResolvedValueOnce([null, config]);
+
+ const result = await getEvaluationConfigById('eval-cfg-1');
+
+ const brandVoice = result.evaluationTypes!.find((t) => t.id === 'brand_voice');
+ expect(brandVoice).toMatchObject({
+ id: 'brand_voice',
+ label: 'Brand Voice',
+ description: 'Checks if the response matches our brand tone',
+ prompt: 'Evaluate whether the response uses our brand voice consistently',
+ isCustom: true,
+ });
+ });
+
+ it('custom types with enabled: false are included but disabled', async () => {
+ const config = makeConfigWithMeta([
+ {
+ id: 'disabled_custom',
+ enabled: false,
+ label: 'Disabled Custom',
+ description: 'Should be present but disabled',
+ },
+ ]);
+ (asaw as jest.Mock).mockResolvedValueOnce([null, config]);
+
+ const result = await getEvaluationConfigById('eval-cfg-1');
+
+ const disabled = result.evaluationTypes!.find((t) => t.id === 'disabled_custom');
+ expect(disabled).toBeDefined();
+ expect(disabled!.enabled).toBe(false);
+ expect(disabled!.isCustom).toBe(true);
+ });
+
+ it('custom types default enabled to false when not specified', async () => {
+ const config = makeConfigWithMeta([
+ { id: 'no_enabled_field', label: 'No Enabled Field', description: 'Test' },
+ ]);
+ (asaw as jest.Mock).mockResolvedValueOnce([null, config]);
+
+ const result = await getEvaluationConfigById('eval-cfg-1');
+
+ const noEnabled = result.evaluationTypes!.find((t) => t.id === 'no_enabled_field');
+ expect(noEnabled).toBeDefined();
+ expect(noEnabled!.enabled).toBe(false);
+ });
+
+ it('custom types use id as label fallback when label is missing', async () => {
+ const config = makeConfigWithMeta([
+ { id: 'fallback_label', description: 'Desc', enabled: true },
+ ]);
+ (asaw as jest.Mock).mockResolvedValueOnce([null, config]);
+
+ const result = await getEvaluationConfigById('eval-cfg-1');
+
+ const fallback = result.evaluationTypes!.find((t) => t.id === 'fallback_label');
+ expect(fallback!.label).toBe('fallback_label');
+ });
+
+ it('custom types use default description when description is missing', async () => {
+ const config = makeConfigWithMeta([
+ { id: 'no_desc', label: 'No Desc', enabled: true },
+ ]);
+ (asaw as jest.Mock).mockResolvedValueOnce([null, config]);
+
+ const result = await getEvaluationConfigById('eval-cfg-1');
+
+ const noDesc = result.evaluationTypes!.find((t) => t.id === 'no_desc');
+ expect(noDesc!.description).toBe('Custom evaluation type');
+ });
+
+ it('built-in types still work correctly alongside custom types', async () => {
+ const config = makeConfigWithMeta([
+ // Override a built-in type
+ { id: 'hallucination', enabled: false, prompt: 'custom hallucination prompt' },
+ // Add a custom type
+ { id: 'custom_one', enabled: true, label: 'Custom One', description: 'Desc 1' },
+ ]);
+ (asaw as jest.Mock).mockResolvedValueOnce([null, config]);
+
+ const result = await getEvaluationConfigById('eval-cfg-1');
+
+ // 11 built-in + 1 custom = 12
+ expect(result.evaluationTypes!.length).toBe(12);
+
+ // Built-in hallucination should have override applied
+ const hallucination = result.evaluationTypes!.find((t) => t.id === 'hallucination');
+ expect(hallucination!.isCustom).toBe(false);
+ expect(hallucination!.enabled).toBe(false);
+ expect(hallucination!.prompt).toBe('custom hallucination prompt');
+
+ // Custom type should be at the end
+ const customOne = result.evaluationTypes!.find((t) => t.id === 'custom_one');
+ expect(customOne!.isCustom).toBe(true);
+ expect(customOne!.enabled).toBe(true);
+ });
+
+ it('built-in type overrides apply enabled state from meta', async () => {
+ const config = makeConfigWithMeta([
+ { id: 'toxicity', enabled: false },
+ { id: 'relevance', enabled: true },
+ ]);
+ (asaw as jest.Mock).mockResolvedValueOnce([null, config]);
+
+ const result = await getEvaluationConfigById('eval-cfg-1');
+
+ const toxicity = result.evaluationTypes!.find((t) => t.id === 'toxicity');
+ expect(toxicity!.enabled).toBe(false); // overridden from true
+
+ const relevance = result.evaluationTypes!.find((t) => t.id === 'relevance');
+ expect(relevance!.enabled).toBe(true); // overridden from false
+ });
+
+ it('built-in types use default prompts from getEvaluationTypeDefaultPrompts', async () => {
+ (getEvaluationTypeDefaultPrompts as jest.Mock).mockResolvedValue({
+ hallucination: 'Default hallucination prompt text',
+ bias: 'Default bias prompt text',
+ });
+ (asaw as jest.Mock).mockResolvedValueOnce([null, makeMockEvalConfig()]);
+
+ const result = await getEvaluationConfigById('eval-cfg-1');
+
+ const hallucination = result.evaluationTypes!.find((t) => t.id === 'hallucination');
+ expect(hallucination!.defaultPrompt).toBe('Default hallucination prompt text');
+
+ const bias = result.evaluationTypes!.find((t) => t.id === 'bias');
+ expect(bias!.defaultPrompt).toBe('Default bias prompt text');
+
+ // Types without a default prompt get empty string
+ const safety = result.evaluationTypes!.find((t) => t.id === 'safety');
+ expect(safety!.defaultPrompt).toBe('');
+ });
+
+ it('custom types always have empty defaultPrompt', async () => {
+ (getEvaluationTypeDefaultPrompts as jest.Mock).mockResolvedValue({
+ my_custom: 'this should not appear',
+ });
+ const config = makeConfigWithMeta([
+ { id: 'my_custom', enabled: true, label: 'My Custom' },
+ ]);
+ (asaw as jest.Mock).mockResolvedValueOnce([null, config]);
+
+ const result = await getEvaluationConfigById('eval-cfg-1');
+
+ const custom = result.evaluationTypes!.find((t) => t.id === 'my_custom');
+ expect(custom!.defaultPrompt).toBe('');
+ });
+
+ it('multiple custom types can coexist', async () => {
+ const config = makeConfigWithMeta([
+ { id: 'custom_a', enabled: true, label: 'Custom A', description: 'First custom' },
+ { id: 'custom_b', enabled: false, label: 'Custom B', description: 'Second custom' },
+ { id: 'custom_c', enabled: true, label: 'Custom C', description: 'Third custom', prompt: 'Prompt C' },
+ ]);
+ (asaw as jest.Mock).mockResolvedValueOnce([null, config]);
+
+ const result = await getEvaluationConfigById('eval-cfg-1');
+
+ // 11 built-in + 3 custom = 14
+ expect(result.evaluationTypes!.length).toBe(14);
+
+ const customTypes = result.evaluationTypes!.filter((t) => t.isCustom);
+ expect(customTypes).toHaveLength(3);
+ expect(customTypes.map((t) => t.id)).toEqual(['custom_a', 'custom_b', 'custom_c']);
+ });
+
+ it('entries with no id are filtered out', async () => {
+ const config = makeConfigWithMeta([
+ { id: '', label: 'Empty ID' },
+ { label: 'No ID at all' } as any,
+ { id: 'valid_custom', enabled: true, label: 'Valid' },
+ ]);
+ (asaw as jest.Mock).mockResolvedValueOnce([null, config]);
+
+ const result = await getEvaluationConfigById('eval-cfg-1');
+
+ // Only 'valid_custom' should be added as custom (entries with no/empty id are filtered)
+ const customTypes = result.evaluationTypes!.filter((t) => t.isCustom);
+ expect(customTypes).toHaveLength(1);
+ expect(customTypes[0].id).toBe('valid_custom');
+ });
+
+ it('rules with valid ruleId are preserved, invalid ones filtered', async () => {
+ const config = makeConfigWithMeta([
+ {
+ id: 'custom_with_rules',
+ enabled: true,
+ label: 'Custom With Rules',
+ rules: [
+ { ruleId: 'rule-1', priority: 1 },
+ { ruleId: '', priority: 2 }, // invalid — empty ruleId
+ { ruleId: 'rule-3', priority: 3 },
+ ],
+ },
+ ]);
+ (asaw as jest.Mock).mockResolvedValueOnce([null, config]);
+
+ const result = await getEvaluationConfigById('eval-cfg-1');
+
+ const custom = result.evaluationTypes!.find((t) => t.id === 'custom_with_rules');
+ expect(custom!.rules).toHaveLength(2);
+ expect(custom!.rules![0].ruleId).toBe('rule-1');
+ expect(custom!.rules![1].ruleId).toBe('rule-3');
+ });
+
+ it('built-in type rules from overrides are applied', async () => {
+ const config = makeConfigWithMeta([
+ {
+ id: 'hallucination',
+ enabled: true,
+ rules: [{ ruleId: 'rule-hal-1', priority: 1 }],
+ },
+ ]);
+ (asaw as jest.Mock).mockResolvedValueOnce([null, config]);
+
+ const result = await getEvaluationConfigById('eval-cfg-1');
+
+ const hallucination = result.evaluationTypes!.find((t) => t.id === 'hallucination');
+ expect(hallucination!.rules).toHaveLength(1);
+ expect(hallucination!.rules![0].ruleId).toBe('rule-hal-1');
+ });
+});
+
+describe('restoreEvaluationCronJobs', () => {
+ beforeEach(() => {
+ jest.clearAllMocks();
+ (getMessage as jest.Mock).mockReturnValue({
+ CRON_JOB_UPDATION_ERROR: 'Cron job error',
+ });
+ (jsonParse as jest.Mock).mockImplementation((v: string) => {
+ try { return JSON.parse(v); } catch { return {}; }
+ });
+ });
+
+ it('does nothing when no auto-evaluation configs exist', async () => {
+ (prisma.evaluationConfigs.findMany as jest.Mock).mockResolvedValue([]);
+ const consoleSpy = jest.spyOn(console, 'log').mockImplementation();
+
+ await restoreEvaluationCronJobs('http://localhost:3000');
+
+ expect(prisma.evaluationConfigs.findMany).toHaveBeenCalledWith({ where: { auto: true } });
+ expect(consoleSpy).toHaveBeenCalledWith('No auto-evaluation configs to restore');
+ consoleSpy.mockRestore();
+ });
+
+ it('does nothing when findMany returns null', async () => {
+ (prisma.evaluationConfigs.findMany as jest.Mock).mockResolvedValue(null);
+ const consoleSpy = jest.spyOn(console, 'log').mockImplementation();
+
+ await restoreEvaluationCronJobs('http://localhost:3000');
+
+ expect(consoleSpy).toHaveBeenCalledWith('No auto-evaluation configs to restore');
+ consoleSpy.mockRestore();
+ });
+
+ it('restores cron jobs for configs with cronJobId and recurringTime', async () => {
+ const mockUpdateCrontab = jest.fn();
+ (Cron as unknown as jest.Mock).mockImplementation(() => ({
+ validateCronSchedule: jest.fn(),
+ updateCrontab: mockUpdateCrontab,
+ deleteCronJob: jest.fn(),
+ }));
+
+ (prisma.evaluationConfigs.findMany as jest.Mock).mockResolvedValue([
+ {
+ id: 'cfg-1',
+ recurringTime: '0 * * * *',
+ meta: JSON.stringify({ cronJobId: 'cron-123' }),
+ },
+ ]);
+ const consoleSpy = jest.spyOn(console, 'log').mockImplementation();
+
+ await restoreEvaluationCronJobs('http://localhost:3000');
+
+ expect(mockUpdateCrontab).toHaveBeenCalledWith(
+ expect.objectContaining({
+ cronId: 'cron-123',
+ cronSchedule: '0 * * * *',
+ cronEnvVars: {
+ EVALUATION_CONFIG_ID: 'cfg-1',
+ API_URL: 'http://localhost:3000',
+ },
+ })
+ );
+ expect(consoleSpy).toHaveBeenCalledWith('Restored cron job for evaluation config cfg-1');
+ consoleSpy.mockRestore();
+ });
+
+ it('skips configs without cronJobId in meta', async () => {
+ const mockUpdateCrontab = jest.fn();
+ (Cron as unknown as jest.Mock).mockImplementation(() => ({
+ validateCronSchedule: jest.fn(),
+ updateCrontab: mockUpdateCrontab,
+ deleteCronJob: jest.fn(),
+ }));
+
+ (prisma.evaluationConfigs.findMany as jest.Mock).mockResolvedValue([
+ {
+ id: 'cfg-2',
+ recurringTime: '0 * * * *',
+ meta: JSON.stringify({}),
+ },
+ ]);
+
+ await restoreEvaluationCronJobs('http://localhost:3000');
+
+ expect(mockUpdateCrontab).not.toHaveBeenCalled();
+ });
+
+ it('skips configs without recurringTime', async () => {
+ const mockUpdateCrontab = jest.fn();
+ (Cron as unknown as jest.Mock).mockImplementation(() => ({
+ validateCronSchedule: jest.fn(),
+ updateCrontab: mockUpdateCrontab,
+ deleteCronJob: jest.fn(),
+ }));
+
+ (prisma.evaluationConfigs.findMany as jest.Mock).mockResolvedValue([
+ {
+ id: 'cfg-3',
+ recurringTime: '',
+ meta: JSON.stringify({ cronJobId: 'cron-456' }),
+ },
+ ]);
+
+ await restoreEvaluationCronJobs('http://localhost:3000');
+
+ expect(mockUpdateCrontab).not.toHaveBeenCalled();
+ });
+
+ it('continues restoring other configs if one fails', async () => {
+ const mockUpdateCrontab = jest.fn()
+ .mockImplementationOnce(() => { throw new Error('cron fail'); })
+ .mockImplementationOnce(() => {});
+ (Cron as unknown as jest.Mock).mockImplementation(() => ({
+ validateCronSchedule: jest.fn(),
+ updateCrontab: mockUpdateCrontab,
+ deleteCronJob: jest.fn(),
+ }));
+
+ (prisma.evaluationConfigs.findMany as jest.Mock).mockResolvedValue([
+ { id: 'cfg-a', recurringTime: '* * * * *', meta: JSON.stringify({ cronJobId: 'c-a' }) },
+ { id: 'cfg-b', recurringTime: '* * * * *', meta: JSON.stringify({ cronJobId: 'c-b' }) },
+ ]);
+
+ const errorSpy = jest.spyOn(console, 'error').mockImplementation();
+ const logSpy = jest.spyOn(console, 'log').mockImplementation();
+
+ await restoreEvaluationCronJobs('http://localhost:3000');
+
+ expect(errorSpy).toHaveBeenCalledWith('Failed to restore cron job for config cfg-a:', expect.any(Error));
+ expect(logSpy).toHaveBeenCalledWith('Restored cron job for evaluation config cfg-b');
+ errorSpy.mockRestore();
+ logSpy.mockRestore();
+ });
+
+ it('catches top-level errors gracefully', async () => {
+ (prisma.evaluationConfigs.findMany as jest.Mock).mockRejectedValue(new Error('DB down'));
+ const errorSpy = jest.spyOn(console, 'error').mockImplementation();
+
+ await restoreEvaluationCronJobs('http://localhost:3000');
+
+ expect(errorSpy).toHaveBeenCalledWith('Failed to restore evaluation cron jobs:', expect.any(Error));
+ errorSpy.mockRestore();
+ });
+});
diff --git a/src/client/src/__tests__/lib/platform/evaluation/rule-engine-context.test.ts b/src/client/src/__tests__/lib/platform/evaluation/rule-engine-context.test.ts
index 6134ad20..d1b0a7eb 100644
--- a/src/client/src/__tests__/lib/platform/evaluation/rule-engine-context.test.ts
+++ b/src/client/src/__tests__/lib/platform/evaluation/rule-engine-context.test.ts
@@ -229,4 +229,5 @@ describe('getContextFromRulesWithPriority', () => {
const result = await getContextFromRulesWithPriority(emptyTrace as any, [{ ruleId: 'r1', priority: 5 }]);
expect(result).toEqual({ contextContents: [], matchingRuleIds: [], contextEntityIds: [] });
});
+
});
diff --git a/src/client/src/__tests__/lib/platform/evaluation/run-evaluation.test.ts b/src/client/src/__tests__/lib/platform/evaluation/run-evaluation.test.ts
index 9bc7d7e5..11c4ecac 100644
--- a/src/client/src/__tests__/lib/platform/evaluation/run-evaluation.test.ts
+++ b/src/client/src/__tests__/lib/platform/evaluation/run-evaluation.test.ts
@@ -223,4 +223,29 @@ describe('runEvaluation — generateText and response parsing', () => {
const [callArgs] = (generateText as jest.Mock).mock.calls[0];
expect(callArgs.temperature).toBe(0);
});
+
+ it('system prompt uses generic TypeA/TypeB examples, not hardcoded eval type names', async () => {
+ await runEvaluation(BASE_PARAMS);
+ const [{ prompt }] = (generateText as jest.Mock).mock.calls[0];
+
+ // The JSON example in the prompt should use generic placeholders
+ expect(prompt).toContain('TypeA');
+ expect(prompt).toContain('TypeB');
+
+ // The prompt template itself should NOT hardcode specific evaluation type names
+ // (the actual type names come from contexts, not from the template)
+ const templateWithoutContexts = prompt.split('Contexts:')[0];
+ expect(templateWithoutContexts).not.toContain('"Hallucination"');
+ expect(templateWithoutContexts).not.toContain('"Bias"');
+ expect(templateWithoutContexts).not.toContain('"Toxicity"');
+ });
+
+ it('system prompt instructs to scan for [X evaluation context] blocks dynamically', async () => {
+ await runEvaluation(BASE_PARAMS);
+ const [{ prompt }] = (generateText as jest.Mock).mock.calls[0];
+
+ // The prompt should instruct the model to scan for evaluation type blocks dynamically
+ expect(prompt).toContain('[X evaluation context]');
+ expect(prompt).toContain('EVERY evaluation type present');
+ });
});
diff --git a/src/client/src/app/(playground)/evaluations/types/[id]/page.tsx b/src/client/src/app/(playground)/evaluations/types/[id]/page.tsx
index 1d31c3d8..8597a469 100644
--- a/src/client/src/app/(playground)/evaluations/types/[id]/page.tsx
+++ b/src/client/src/app/(playground)/evaluations/types/[id]/page.tsx
@@ -1,6 +1,6 @@
"use client";
-import { useParams } from "next/navigation";
+import { useParams, useRouter } from "next/navigation";
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
import { Badge } from "@/components/ui/badge";
import { Switch } from "@/components/ui/switch";
@@ -16,7 +16,7 @@ import { useEffect, useState } from "react";
import { EVALUATION_TYPES } from "@/constants/evaluation-types";
import { Rule } from "@/types/rule-engine";
import { toast } from "sonner";
-import { Link2, Plus, Trash2, ExternalLink, ArrowLeft } from "lucide-react";
+import { Link2, Plus, Trash2, ExternalLink, ArrowLeft, Sparkles } from "lucide-react";
import { Button } from "@/components/ui/button";
import Link from "next/link";
@@ -28,6 +28,9 @@ interface RuleWithPriority {
interface EvaluationTypeConfig {
id: string;
enabled: boolean;
+ isCustom?: boolean;
+ label?: string;
+ description?: string;
rules: RuleWithPriority[];
prompt?: string;
defaultPrompt?: string;
@@ -35,6 +38,7 @@ interface EvaluationTypeConfig {
export default function EvaluationTypeDetailPage() {
const params = useParams();
+ const router = useRouter();
const typeId = params.id as string;
const et = EVALUATION_TYPES.find((e) => e.id === typeId);
@@ -47,6 +51,7 @@ export default function EvaluationTypeDetailPage() {
Array<{ rule_id: string; entity_id: string }>
>();
const { fireRequest: saveType, isLoading: isSaving } = useFetchWrapper();
+ const { fireRequest: deleteType, isLoading: isDeleting } = useFetchWrapper();
const rulesLinkedFromRulePage = (evalEntities || [])
.filter((e) => e.entity_id === typeId)
@@ -72,6 +77,9 @@ export default function EvaluationTypeDetailPage() {
setConfig({
id: data.id,
enabled: data.enabled ?? false,
+ isCustom: data.isCustom,
+ label: data.label,
+ description: data.description,
rules: data.rules || [],
prompt: data.prompt,
defaultPrompt: data.defaultPrompt,
@@ -131,11 +139,16 @@ export default function EvaluationTypeDetailPage() {
const handleSave = () => {
if (!config) return;
- const payload = {
+ const payload: Record<string, any> = {
enabled: config.enabled,
rules: config.rules.filter((r) => r.ruleId),
prompt: config.prompt?.trim() || undefined,
};
+ if (config.isCustom) {
+ payload.isCustom = true;
+ payload.label = config.label;
+ payload.description = config.description;
+ }
saveType({
requestType: "PATCH",
url: `/api/evaluation/types/${typeId}`,
@@ -157,15 +170,29 @@ export default function EvaluationTypeDetailPage() {
});
};
- if (!et) {
+ const handleDelete = () => {
+ if (!confirm(`Delete custom evaluation type "${displayLabel}"? This cannot be undone.`)) return;
+ deleteType({
+ requestType: "DELETE",
+ url: `/api/evaluation/types/${typeId}`,
+ responseDataKey: "data",
+ successCb: () => {
+ toast.success("Custom evaluation type deleted");
+ router.push("/evaluations/types");
+ },
+ failureCb: (err?: string) => toast.error(err || "Failed to delete"),
+ });
+ };
+
+ // Derive display label/description from built-in constant or custom type from API
+ const isCustom = !et && config?.isCustom;
+ const displayLabel = et?.label || config?.label || typeId;
+ const displayDescription = et?.description || config?.description || "";
+
+ if (!et && !config) {
return (
<div className="flex flex-1 h-full w-full p-6 items-center justify-center">
- <p className="text-stone-500">Evaluation type not found</p>
- <Link href="/evaluations/types">
- <Button variant="outline" className="ml-4">
- Back to types
- </Button>
- </Link>
+ <p className="text-stone-500">Loading...</p>
</div>
);
}
@@ -184,11 +211,17 @@ export default function EvaluationTypeDetailPage() {
</Button>
</Link>
<div>
- <h2 className="text-lg font-semibold text-stone-800 dark:text-stone-200">
- {et.label}
+ <h2 className="text-lg font-semibold text-stone-800 dark:text-stone-200 flex items-center gap-2">
+ {displayLabel}
+ {isCustom && (
+ <Badge variant="outline" className="text-xs font-normal gap-1 border-primary/30 text-primary">
+ <Sparkles className="size-3" />
+ Custom
+ </Badge>
+ )}
</h2>
<p className="text-sm text-stone-500 dark:text-stone-400">
- {et.description}
+ {displayDescription}
</p>
</div>
</div>
@@ -199,7 +232,7 @@ export default function EvaluationTypeDetailPage() {
<CardHeader className="pb-4">
<CardTitle className="text-base">Purpose</CardTitle>
<p className="text-sm text-stone-500 dark:text-stone-400 font-normal">
- {et.description}
+ {displayDescription}
</p>
</CardHeader>
</Card>
@@ -214,7 +247,7 @@ export default function EvaluationTypeDetailPage() {
</CardHeader>
<CardContent>
<textarea
- className="w-full min-h-[120px] text-xs bg-stone-50 dark:bg-stone-900/50 p-3 rounded-lg border border-stone-200 dark:border-stone-700 font-mono resize-y"
+ className="w-full min-h-[120px] text-xs bg-stone-50 dark:bg-stone-900/50 text-stone-900 dark:text-stone-100 p-3 rounded-lg border border-stone-200 dark:border-stone-700 font-mono resize-y placeholder:text-stone-400 dark:placeholder:text-stone-500 focus:outline-none focus:ring-2 focus:ring-primary/50 focus:border-primary/50"
value={
config?.prompt ?? config?.defaultPrompt ?? ""
}
@@ -338,6 +371,17 @@ export default function EvaluationTypeDetailPage() {
<Button onClick={handleSave} disabled={isSaving} className="w-full sm:w-auto bg-primary dark:bg-primary text-white dark:text-white hover:bg-primary/90 dark:hover:bg-primary/90">
{isSaving ? "Saving..." : "Save Changes"}
</Button>
+ {isCustom && (
+ <Button
+ variant="outline"
+ onClick={handleDelete}
+ disabled={isDeleting}
+ className="w-full sm:w-auto text-red-600 dark:text-red-400 border-red-200 dark:border-red-800 hover:bg-red-50 dark:hover:bg-red-950/30 hover:text-red-700 dark:hover:text-red-300"
+ >
+ <Trash2 className="size-4 mr-1.5" />
+ {isDeleting ? "Deleting..." : "Delete Custom Type"}
+ </Button>
+ )}
{rulesLinkedFromRulePage.length > 0 && (
<Card className="border-stone-200 dark:border-stone-800 shadow-sm">
<CardHeader className="pb-2">
diff --git a/src/client/src/app/(playground)/evaluations/types/new/page.tsx b/src/client/src/app/(playground)/evaluations/types/new/page.tsx
new file mode 100644
index 00000000..1e363b25
--- /dev/null
+++ b/src/client/src/app/(playground)/evaluations/types/new/page.tsx
@@ -0,0 +1,211 @@
+"use client";
+
+import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
+import { Input } from "@/components/ui/input";
+import { Label } from "@/components/ui/label";
+import { Button } from "@/components/ui/button";
+import { ArrowLeft, Sparkles, Info } from "lucide-react";
+import Link from "next/link";
+import { useRouter } from "next/navigation";
+import { useState } from "react";
+import { toast } from "sonner";
+import useFetchWrapper from "@/utils/hooks/useFetchWrapper";
+
+export default function CreateCustomEvaluationTypePage() {
+ const router = useRouter();
+ const [typeId, setTypeId] = useState("");
+ const [label, setLabel] = useState("");
+ const [description, setDescription] = useState("");
+ const [prompt, setPrompt] = useState("");
+
+ const { fireRequest: getTypes } = useFetchWrapper();
+ const { fireRequest: saveTypes, isLoading: isSaving } = useFetchWrapper();
+
+ const handleCreate = async () => {
+ const id = typeId.trim().toLowerCase().replace(/[^a-z0-9_]/g, "_");
+ if (!id) {
+ toast.error("Type ID is required");
+ return;
+ }
+ if (!label.trim()) {
+ toast.error("Label is required");
+ return;
+ }
+ if (!prompt.trim()) {
+ toast.error("Evaluation prompt is required");
+ return;
+ }
+
+ // Fetch existing types to append to
+ getTypes({
+ requestType: "GET",
+ url: "/api/evaluation/types",
+ responseDataKey: "data",
+ successCb: (existing: any) => {
+ const existingTypes = Array.isArray(existing) ? existing : [];
+ if (existingTypes.some((t: any) => t.id === id)) {
+ toast.error("An evaluation type with this ID already exists");
+ return;
+ }
+
+ const updatedTypes = [
+ ...existingTypes.map((t: any) => ({
+ id: t.id,
+ enabled: t.enabled,
+ isCustom: t.isCustom,
+ label: t.isCustom ? t.label : undefined,
+ description: t.isCustom ? t.description : undefined,
+ prompt: t.prompt,
+ rules: t.rules,
+ })),
+ {
+ id,
+ enabled: true,
+ isCustom: true,
+ label: label.trim(),
+ description: description.trim() || "Custom evaluation type",
+ prompt: prompt.trim(),
+ rules: [],
+ },
+ ];
+
+ saveTypes({
+ requestType: "POST",
+ url: "/api/evaluation/types",
+ body: JSON.stringify({ types: updatedTypes }),
+ responseDataKey: "data",
+ successCb: () => {
+ toast.success(`Custom evaluation type "${label.trim()}" created`);
+ router.push(`/evaluations/types/${id}`);
+ },
+ failureCb: (err?: string) =>
+ toast.error(err || "Failed to create custom type"),
+ });
+ },
+ failureCb: (err?: string) =>
+ toast.error(err || "Failed to load existing types"),
+ });
+ };
+
+ return (
+ <div className="flex flex-col flex-1 h-full w-full p-6 overflow-auto gap-6">
+ {/* Header */}
+ <div className="flex items-center gap-4">
+ <Link href="/evaluations/types">
+ <Button
+ variant="outline"
+ size="icon"
+ className="shrink-0 text-stone-700 dark:text-stone-200 border-stone-300 dark:border-stone-600 hover:bg-stone-100 dark:hover:bg-stone-800"
+ >
+ <ArrowLeft className="size-4" />
+ </Button>
+ </Link>
+ <div>
+ <h2 className="text-lg font-semibold text-stone-900 dark:text-stone-100 flex items-center gap-2">
+ <Sparkles className="size-5 text-primary" />
+ Create Custom Evaluation Type
+ </h2>
+ <p className="text-sm text-stone-500 dark:text-stone-400">
+ Define a custom evaluation type with your own prompt. The LLM judge will use it to evaluate traces.
+ </p>
+ </div>
+ </div>
+
+ <div className="grid grid-cols-3 gap-4">
+ <div className="col-span-2 space-y-4">
+ {/* Type ID & Label */}
+ <Card className="border-stone-200 dark:border-stone-800 shadow-sm">
+ <CardHeader className="pb-2">
+ <CardTitle className="text-base text-stone-900 dark:text-stone-100">Type Details</CardTitle>
+ </CardHeader>
+ <CardContent className="space-y-4">
+ <div className="grid gap-2">
+ <Label htmlFor="type-id" className="text-stone-700 dark:text-stone-300">Type ID</Label>
+ <Input
+ id="type-id"
+ placeholder="e.g. domain_accuracy"
+ value={typeId}
+ onChange={(e) =>
+ setTypeId(
+ e.target.value
+ .toLowerCase()
+ .replace(/[^a-z0-9_]/g, "_")
+ )
+ }
+ />
+ <p className="text-xs text-stone-500 dark:text-stone-400">
+ Lowercase letters, numbers, and underscores only. Used as the internal identifier.
+ </p>
+ </div>
+ <div className="grid gap-2">
+ <Label htmlFor="type-label" className="text-stone-700 dark:text-stone-300">Label</Label>
+ <Input
+ id="type-label"
+ placeholder="e.g. Domain Accuracy"
+ value={label}
+ onChange={(e) => setLabel(e.target.value)}
+ />
+ </div>
+ <div className="grid gap-2">
+ <Label htmlFor="type-description" className="text-stone-700 dark:text-stone-300">Description</Label>
+ <Input
+ id="type-description"
+ placeholder="e.g. Evaluates responses against domain-specific knowledge"
+ value={description}
+ onChange={(e) => setDescription(e.target.value)}
+ />
+ </div>
+ </CardContent>
+ </Card>
+
+ {/* Evaluation Prompt */}
+ <Card className="border-stone-200 dark:border-stone-800 shadow-sm">
+ <CardHeader className="pb-2">
+ <CardTitle className="text-base text-stone-900 dark:text-stone-100">Evaluation Prompt</CardTitle>
+ <p className="text-xs text-stone-500 dark:text-stone-400 font-normal">
+ The LLM judge uses this prompt to evaluate each trace. Start with a [Label evaluation context] header.
+ </p>
+ </CardHeader>
+ <CardContent>
+ <textarea
+ id="type-prompt"
+ className="w-full min-h-[200px] text-sm bg-stone-50 dark:bg-stone-900/50 text-stone-900 dark:text-stone-100 p-3 rounded-lg border border-stone-200 dark:border-stone-700 font-mono resize-y placeholder:text-stone-400 dark:placeholder:text-stone-500 focus:outline-none focus:ring-2 focus:ring-primary/50 focus:border-primary/50"
+ placeholder={`[Domain Accuracy evaluation context]\nConsider: whether the response aligns with domain-specific knowledge and terminology.\nLook for incorrect use of domain terms, inaccurate domain-specific claims, and deviations from established domain practices.`}
+ value={prompt}
+ onChange={(e) => setPrompt(e.target.value)}
+ />
+ </CardContent>
+ </Card>
+ </div>
+
+ {/* Sidebar */}
+ <div className="space-y-4">
+ <Button
+ onClick={handleCreate}
+ disabled={isSaving}
+ className="w-full bg-primary dark:bg-primary text-white dark:text-white hover:bg-primary/90 dark:hover:bg-primary/90"
+ >
+ {isSaving ? "Creating..." : "Create Evaluation Type"}
+ </Button>
+
+ <Card className="border-blue-200 dark:border-blue-900/40 bg-blue-50/50 dark:bg-blue-950/20 shadow-sm">
+ <CardContent className="pt-4">
+ <div className="flex gap-2">
+ <Info className="size-4 text-blue-600 dark:text-blue-400 shrink-0 mt-0.5" />
+ <div className="text-xs text-blue-900 dark:text-blue-100 space-y-2">
+ <p className="font-medium">How it works</p>
+ <p className="text-blue-700 dark:text-blue-300">
+ Custom types run alongside built-in evaluations (hallucination, bias, toxicity, etc.). The LLM judge produces a score, classification, and explanation for each type.
+ </p>
+ <p className="text-blue-700 dark:text-blue-300">
+ After creating, you can link Rule Engine rules to conditionally apply this evaluation based on trace attributes.
+ </p>
+ </div>
+ </div>
+ </CardContent>
+ </Card>
+ </div>
+ </div>
+ </div>
+ );
+}
diff --git a/src/client/src/app/(playground)/evaluations/types/page.tsx b/src/client/src/app/(playground)/evaluations/types/page.tsx
index 242547aa..65f155a3 100644
--- a/src/client/src/app/(playground)/evaluations/types/page.tsx
+++ b/src/client/src/app/(playground)/evaluations/types/page.tsx
@@ -1,32 +1,36 @@
"use client";
-import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
+import { Card, CardHeader, CardTitle } from "@/components/ui/card";
import { Badge } from "@/components/ui/badge";
import useFetchWrapper from "@/utils/hooks/useFetchWrapper";
import { useEffect, useState } from "react";
import { EVALUATION_TYPES } from "@/constants/evaluation-types";
import { Rule } from "@/types/rule-engine";
-import { Layers, CheckCircle2, ChevronRight, ArrowLeft } from "lucide-react";
+import { Layers, CheckCircle2, ChevronRight, ArrowLeft, Plus, Sparkles } from "lucide-react";
import Link from "next/link";
import { Button } from "@/components/ui/button";
-interface EvaluationTypeConfig {
+interface EvaluationTypeDisplay {
id: string;
+ label: string;
+ description: string;
+ enabledByDefault: boolean;
enabled: boolean;
+ isCustom?: boolean;
rules?: Array<{ ruleId: string; priority: number }>;
+ prompt?: string;
}
export default function EvaluationTypesPage() {
const { fireRequest: getTypes, data: typesResponse } = useFetchWrapper<
- EvaluationTypeConfig[] | { data: any[] }
+ EvaluationTypeDisplay[] | { data: any[] }
>();
const { fireRequest: getRules, data: rules } = useFetchWrapper<Rule[]>();
const { fireRequest: getEvalEntities, data: evalEntities } = useFetchWrapper<
Array<{ rule_id: string; entity_id: string }>
>();
- const [typeConfigs, setTypeConfigs] = useState<
- Record<string, { enabled: boolean; rulesCount: number }>
- >({});
+
+ const [allTypes, setAllTypes] = useState<EvaluationTypeDisplay[]>([]);
const rulesLinkedToType = (evalEntities || []).reduce<
Record<string, string[]>
@@ -54,19 +58,96 @@ export default function EvaluationTypesPage() {
? typesResponse
: (typesResponse as { data?: any[] })?.data;
if (savedTypes !== undefined && Array.isArray(savedTypes)) {
- const map: Record<string, { enabled: boolean; rulesCount: number }> = {};
- for (const et of EVALUATION_TYPES) {
- const existing = savedTypes.find((t: any) => t.id === et.id);
- const rules = existing?.rules || [];
- map[et.id] = {
- enabled: existing?.enabled ?? et.enabledByDefault,
- rulesCount: Array.isArray(rules) ? rules.filter((r: any) => r?.ruleId).length : 0,
- };
- }
- setTypeConfigs(map);
+ setAllTypes(savedTypes);
}
}, [typesResponse]);
+ const builtInIds = new Set<string>(EVALUATION_TYPES.map((et) => et.id));
+ const builtInTypes = allTypes.filter((t) => builtInIds.has(t.id));
+ const customTypes = allTypes.filter((t) => !builtInIds.has(t.id));
+
+ const displayBuiltIn = EVALUATION_TYPES.map((et) => {
+ const saved = builtInTypes.find((t) => t.id === et.id);
+ return {
+ ...et,
+ enabled: saved?.enabled ?? et.enabledByDefault,
+ isCustom: false,
+ rules: saved?.rules || [],
+ };
+ });
+
+ const renderTypeCard = (type: EvaluationTypeDisplay, isCustomType: boolean) => {
+ const hasRules =
+ (type.rules?.length ?? 0) > 0 ||
+ (rulesLinkedToType[type.id]?.length ?? 0) > 0;
+ return (
+ <Link key={type.id} href={`/evaluations/types/${type.id}`}>
+ <Card className={`shadow-sm overflow-hidden transition-colors cursor-pointer ${
+ isCustomType
+ ? "border-primary/20 dark:border-primary/15 hover:border-primary/40 dark:hover:border-primary/30"
+ : "border-stone-200 dark:border-stone-800 hover:border-stone-300 dark:hover:border-stone-700"
+ }`}>
+ <CardHeader className="pb-3">
+ <div className="flex items-start justify-between gap-4">
+ <div className="flex-1 min-w-0">
+ <CardTitle className="text-base flex items-center gap-2 flex-wrap text-stone-900 dark:text-stone-100">
+ {type.label}
+ {isCustomType && (
+ <Badge variant="outline" className="text-xs font-normal gap-1 shrink-0 border-primary/30 text-primary dark:border-primary/20">
+ <Sparkles className="size-3" />
+ Custom
+ </Badge>
+ )}
+ {hasRules && (
+ <Badge
+ variant="outline"
+ className="text-xs font-normal gap-1 shrink-0"
+ >
+ <CheckCircle2 className="size-3" />
+ Rule engine
+ </Badge>
+ )}
+ </CardTitle>
+ <p className="text-sm text-stone-500 dark:text-stone-400 mt-1">
+ {type.description}
+ </p>
+ {(rulesLinkedToType[type.id]?.length ?? 0) > 0 && (
+ <div className="flex items-center gap-1.5 mt-2 flex-wrap">
+ <span className="text-xs text-stone-500 dark:text-stone-400">
+ Linked from rules:
+ </span>
+ {rulesLinkedToType[type.id].slice(0, 3).map((rid) => {
+ const r = (rules || []).find((x) => x.id === rid);
+ return (
+ <span
+ key={rid}
+ className="inline-flex items-center gap-1 text-xs text-primary"
+ >
+ {r?.name || rid.slice(0, 8)}
+ </span>
+ );
+ })}
+ {rulesLinkedToType[type.id].length > 3 && (
+ <span className="text-xs text-stone-400 dark:text-stone-500">
+ +{rulesLinkedToType[type.id].length - 3} more
+ </span>
+ )}
+ </div>
+ )}
+ </div>
+ <div className="flex items-center gap-2 shrink-0">
+ <Badge variant={type.enabled ? "default" : "secondary"}>
+ {type.enabled ? "Enabled" : "Disabled"}
+ </Badge>
+ <ChevronRight className="size-4 text-stone-400 dark:text-stone-500" />
+ </div>
+ </div>
+ </CardHeader>
+ </Card>
+ </Link>
+ );
+ };
+
return (
<div className="flex flex-1 h-full w-full p-6 overflow-auto">
<div className="w-full space-y-6">
@@ -80,87 +161,43 @@ export default function EvaluationTypesPage() {
<ArrowLeft className="size-4" />
</Button>
</Link>
- <div>
- <h2 className="text-lg font-semibold text-stone-800 dark:text-stone-200 flex items-center gap-2">
+ <div className="flex-1">
+ <h2 className="text-lg font-semibold text-stone-900 dark:text-stone-100 flex items-center gap-2">
<Layers className="size-5" />
Evaluation Types
</h2>
<p className="text-sm text-stone-500 dark:text-stone-400 mt-1">
- Configure each evaluation type individually. Click a type to set rules,
- priorities, and prebuilt context.
+ Configure built-in evaluation types or create custom ones. Click a type to set rules,
+ priorities, and evaluation context.
</p>
</div>
+ <Link href="/evaluations/types/new">
+ <Button className="shrink-0">
+ <Plus className="size-4 mr-1.5" />
+ Create Custom Type
+ </Button>
+ </Link>
</div>
+ {/* Built-in types */}
<div className="grid grid-cols-1 md:grid-cols-2 gap-4">
- {EVALUATION_TYPES.map((et) => {
- const config = typeConfigs[et.id] ?? {
- enabled: et.enabledByDefault,
- rulesCount: 0,
- };
- const hasRules =
- config.rulesCount > 0 ||
- (rulesLinkedToType[et.id]?.length ?? 0) > 0;
- return (
- <Link key={et.id} href={`/evaluations/types/${et.id}`}>
- <Card className="border-stone-200 dark:border-stone-800 shadow-sm overflow-hidden hover:border-stone-300 dark:hover:border-stone-700 transition-colors cursor-pointer">
- <CardHeader className="pb-3">
- <div className="flex items-start justify-between gap-4">
- <div className="flex-1 min-w-0">
- <CardTitle className="text-base flex items-center gap-2 flex-wrap">
- {et.label}
- {hasRules && (
- <Badge
- variant="outline"
- className="text-xs font-normal gap-1 shrink-0"
- >
- <CheckCircle2 className="size-3" />
- Rule engine
- </Badge>
- )}
- </CardTitle>
- <p className="text-sm text-stone-500 dark:text-stone-400 mt-1">
- {et.description}
- </p>
- {(rulesLinkedToType[et.id]?.length ?? 0) > 0 && (
- <div className="flex items-center gap-1.5 mt-2 flex-wrap">
- <span className="text-xs text-stone-500 dark:text-stone-400">
- Linked from rules:
- </span>
- {rulesLinkedToType[et.id].slice(0, 3).map((rid) => {
- const r = (rules || []).find((x) => x.id === rid);
- return (
- <span
- key={rid}
- className="inline-flex items-center gap-1 text-xs text-primary"
- >
- {r?.name || rid.slice(0, 8)}
- </span>
- );
- })}
- {rulesLinkedToType[et.id].length > 3 && (
- <span className="text-xs text-stone-400">
- +{rulesLinkedToType[et.id].length - 3} more
- </span>
- )}
- </div>
- )}
- </div>
- <div className="flex items-center gap-2 shrink-0">
- <Badge
- variant={config.enabled ? "default" : "secondary"}
- >
- {config.enabled ? "Enabled" : "Disabled"}
- </Badge>
- <ChevronRight className="size-4 text-stone-400" />
- </div>
- </div>
- </CardHeader>
- </Card>
- </Link>
- );
- })}
+ {displayBuiltIn.map((et) => renderTypeCard(et, false))}
</div>
+
+ {/* Custom types */}
+ {customTypes.length > 0 && (
+ <>
+ <div className="flex items-center gap-2 pt-2">
+ <Sparkles className="size-4 text-primary" />
+ <h3 className="text-sm font-semibold text-stone-700 dark:text-stone-300">
+ Custom Evaluation Types
+ </h3>
+ </div>
+ <div className="grid grid-cols-1 md:grid-cols-2 gap-4">
+ {customTypes.map((ct) => renderTypeCard(ct, true))}
+ </div>
+ </>
+ )}
</div>
</div>
);
diff --git a/src/client/src/app/api/evaluation/types/[id]/route.ts b/src/client/src/app/api/evaluation/types/[id]/route.ts
index 5bf9276d..cf735f6f 100644
--- a/src/client/src/app/api/evaluation/types/[id]/route.ts
+++ b/src/client/src/app/api/evaluation/types/[id]/route.ts
@@ -12,11 +12,12 @@ export interface RuleWithPriority {
export interface EvaluationTypeConfig {
id: string;
enabled: boolean;
+ isCustom?: boolean;
+ label?: string;
+ description?: string;
rules?: RuleWithPriority[];
prompt?: string;
defaultPrompt?: string;
- label?: string;
- description?: string;
}
function normalizeRules(rules: any[]): RuleWithPriority[] {
@@ -77,6 +78,12 @@ export async function PATCH(
rules: body.rules !== undefined ? normalizeRules(body.rules) : existing.rules || [],
prompt: body.prompt !== undefined ? body.prompt : existing.prompt,
};
+ // Preserve or update custom type metadata
+ if (body.isCustom || (existing as any).isCustom) {
+ updated.isCustom = true;
+ updated.label = body.label ?? (existing as any).label ?? typeId;
+ updated.description = body.description ?? (existing as any).description ?? "";
+ }
if (idx >= 0) {
types[idx] = updated;
} else {
@@ -90,3 +97,37 @@ export async function PATCH(
await syncRuleEntitiesFromConfig();
return Response.json({ data: updated });
}
+
+export async function DELETE(
+ _: NextRequest,
+ { params }: { params: { id: string } }
+) {
+ const typeId = params.id;
+ const [err, config] = await asaw(getEvaluationConfig(undefined, true, false));
+ if (err || !config?.id) {
+ return Response.json(
+ { err: "Evaluation config not found" },
+ { status: 400 }
+ );
+ }
+ const prisma = (await import("@/lib/prisma")).default;
+ const meta = jsonParse((config as any).meta || "{}") as Record<string, any>;
+ const types: EvaluationTypeConfig[] =
+ (meta.evaluationTypes as EvaluationTypeConfig[]) || [];
+
+ const typeToDelete = types.find((t) => t.id === typeId);
+ if (!typeToDelete?.isCustom) {
+ return Response.json(
+ { err: "Only custom evaluation types can be deleted" },
+ { status: 400 }
+ );
+ }
+
+ meta.evaluationTypes = types.filter((t) => t.id !== typeId);
+ await prisma.evaluationConfigs.update({
+ where: { id: config.id },
+ data: { meta: jsonStringify(meta) },
+ });
+ await syncRuleEntitiesFromConfig();
+ return Response.json({ data: { deleted: typeId } });
+}
diff --git a/src/client/src/app/api/evaluation/types/route.ts b/src/client/src/app/api/evaluation/types/route.ts
index 320cd6ba..42757396 100644
--- a/src/client/src/app/api/evaluation/types/route.ts
+++ b/src/client/src/app/api/evaluation/types/route.ts
@@ -12,11 +12,12 @@ export interface RuleWithPriority {
export interface EvaluationTypeConfig {
id: string;
enabled: boolean;
+ isCustom?: boolean;
+ label?: string;
+ description?: string;
rules?: RuleWithPriority[];
ruleId?: string;
priority?: number;
- label?: string;
- description?: string;
defaultPrompt?: string;
prompt?: string;
}
@@ -30,11 +31,21 @@ function normalizeTypeConfig(t: any): EvaluationTypeConfig {
: t.ruleId
? [{ ruleId: t.ruleId, priority: Number(t.priority) || 0 }]
: [];
- return {
+ const config: EvaluationTypeConfig = {
id: t.id,
enabled: !!t.enabled,
rules,
};
+ // Preserve custom type metadata
+ if (t.isCustom) {
+ config.isCustom = true;
+ if (t.label) config.label = String(t.label).trim();
+ if (t.description) config.description = String(t.description).trim();
+ if (t.prompt) config.prompt = String(t.prompt).trim();
+ } else if (t.prompt) {
+ config.prompt = String(t.prompt).trim();
+ }
+ return config;
}
export async function GET(_: NextRequest) {
diff --git a/src/client/src/components/(playground)/rule-engine/rule-entities-card.tsx b/src/client/src/components/(playground)/rule-engine/rule-entities-card.tsx
index f96cd152..99dc715c 100644
--- a/src/client/src/components/(playground)/rule-engine/rule-entities-card.tsx
+++ b/src/client/src/components/(playground)/rule-engine/rule-entities-card.tsx
@@ -123,10 +123,27 @@ export default function RuleEntitiesCard({ ruleId }: { ruleId: string }) {
}
setIsFetchingOptions(true);
if (type === "evaluation") {
- setEntityOptions(
- EVALUATION_TYPES.map((e) => ({ id: e.id, name: e.label }))
- );
- setIsFetchingOptions(false);
+ // Built-in types from constants
+ const builtInOptions = EVALUATION_TYPES.map((e) => ({ id: e.id, name: e.label }));
+ // Also fetch custom types from API
+ fireEntityOptions({
+ requestType: "GET",
+ url: "/api/evaluation/types",
+ responseDataKey: "data",
+ successCb: (data: any) => {
+ const apiTypes = Array.isArray(data) ? data : [];
+ const builtInIds = new Set(EVALUATION_TYPES.map((e) => e.id));
+ const customOptions = apiTypes
+ .filter((t: any) => t.isCustom && !builtInIds.has(t.id))
+ .map((t: any) => ({ id: t.id, name: t.label || t.id }));
+ setEntityOptions([...builtInOptions, ...customOptions]);
+ setIsFetchingOptions(false);
+ },
+ failureCb: () => {
+ setEntityOptions(builtInOptions);
+ setIsFetchingOptions(false);
+ },
+ });
} else if (type === "context") {
fireEntityOptions({
requestType: "GET",
diff --git a/src/client/src/lib/platform/evaluation/config.ts b/src/client/src/lib/platform/evaluation/config.ts
index ba027e19..66d058cd 100644
--- a/src/client/src/lib/platform/evaluation/config.ts
+++ b/src/client/src/lib/platform/evaluation/config.ts
@@ -25,6 +25,7 @@ export interface EvaluationTypeWithPrompt {
description: string;
enabledByDefault: boolean;
enabled: boolean;
+ isCustom?: boolean;
rules?: Array<{ ruleId: string; priority: number }>;
prompt?: string;
defaultPrompt: string;
@@ -37,6 +38,9 @@ async function buildEvaluationTypesWithPrompts(
const userOverrides = (meta.evaluationTypes as Array<{
id: string;
enabled?: boolean;
+ label?: string;
+ description?: string;
+ isCustom?: boolean;
rules?: Array<{ ruleId: string; priority: number }>;
prompt?: string;
}>) || [];
@@ -45,7 +49,10 @@ async function buildEvaluationTypesWithPrompts(
userOverrides.filter((t) => t?.id).map((t) => [t.id, t])
);
- return EVALUATION_TYPES.map((et) => {
+ const builtInIds = new Set(EVALUATION_TYPES.map((et) => et.id));
+
+ // Built-in types merged with overrides
+ const builtInTypes: EvaluationTypeWithPrompt[] = EVALUATION_TYPES.map((et) => {
const override = overrideMap.get(et.id);
return {
id: et.id,
@@ -53,11 +60,29 @@ async function buildEvaluationTypesWithPrompts(
description: et.description,
enabledByDefault: et.enabledByDefault,
enabled: override?.enabled ?? et.enabledByDefault,
+ isCustom: false,
rules: override?.rules?.filter((r) => r?.ruleId) ?? [],
prompt: override?.prompt,
defaultPrompt: defaultPrompts[et.id] ?? "",
};
});
+
+ // Custom types from user meta (not in EVALUATION_TYPES constant)
+ const customTypes: EvaluationTypeWithPrompt[] = userOverrides
+ .filter((t) => t?.id && !builtInIds.has(t.id as any))
+ .map((t) => ({
+ id: t.id,
+ label: t.label || t.id,
+ description: t.description || "Custom evaluation type",
+ enabledByDefault: false,
+ enabled: t.enabled ?? false,
+ isCustom: true,
+ rules: t.rules?.filter((r) => r?.ruleId) ?? [],
+ prompt: t.prompt,
+ defaultPrompt: "",
+ }));
+
+ return [...builtInTypes, ...customTypes];
}
export async function getEvaluationConfig(