🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { z } from "zod";
import { isPresent } from "../utils/typeChecks";
// Category type, used for categorical and boolean configs
export const ScoreConfigCategory = z.object({
label: z.string().min(1),
value: z.number(),
});
export const SCORE_CONFIG_NAME_MIN_LENGTH = 1;
export const SCORE_CONFIG_NAME_MAX_LENGTH = 80;
const SCORE_CONFIG_NAME_MAX_LENGTH_FOR_DB_READ = 80;
/** Input-only schema for score config names. Use at API/tRPC boundaries, not for DB reads. */
export const ScoreConfigNameSchema = z
.string()
.min(SCORE_CONFIG_NAME_MIN_LENGTH)
.max(SCORE_CONFIG_NAME_MAX_LENGTH)
.regex(/^[\p{L}\p{N}_ .()-]+$/u, "Name contains invalid characters");
// Numeric config fields
export const NumericConfigFields = z.object({
maxValue: z.number().nullish(),
minValue: z.number().nullish(),
dataType: z.literal("NUMERIC"),
categories: z.undefined().nullish(),
});
// Boolean config fields
export const BooleanConfigFields = z.object({
dataType: z.literal("BOOLEAN"),
maxValue: z.number().nullish(),
minValue: z.number().nullish(),
categories: z
.array(ScoreConfigCategory)
.length(2, "Boolean data type must have exactly 2 categories.")
.refine((categories) => {
const expectedCategories = [
{ label: "True", value: 1 },
{ label: "False", value: 0 },
];
return categories.every(
(category, index) =>
category.label === expectedCategories[index].label &&
category.value === expectedCategories[index].value,
);
}),
});
// Category config fields and types
export const validateCategories = (
categories: z.infer<typeof ScoreConfigCategory>[],
ctx: z.RefinementCtx,
) => {
const uniqueNames = new Set<string>();
const uniqueValues = new Set<number>();
for (const category of categories) {
if (uniqueNames.has(category.label)) {
ctx.addIssue({
code: "custom",
message: `Duplicate category label: ${category.label}, category labels must be unique`,
});
return;
}
uniqueNames.add(category.label);
if (uniqueValues.has(category.value)) {
ctx.addIssue({
code: "custom",
message: `Duplicate category value: ${category.value}, category values must be unique`,
});
return;
}
uniqueValues.add(category.value);
}
};
export const CategoricalConfigFields = z.object({
maxValue: z.undefined().nullish(),
minValue: z.undefined().nullish(),
dataType: z.literal("CATEGORICAL"),
categories: z.array(ScoreConfigCategory).superRefine(validateCategories),
});
export const TextConfigFields = z.object({
maxValue: z.undefined().nullish(),
minValue: z.undefined().nullish(),
dataType: z.literal("TEXT"),
categories: z.undefined().nullish(),
});
const ScoreConfigBase = z.object({
id: z.string(),
name: z.string().min(1).max(SCORE_CONFIG_NAME_MAX_LENGTH_FOR_DB_READ),
isArchived: z.boolean(),
description: z.string().nullish(),
createdAt: z.coerce.date(),
updatedAt: z.coerce.date(),
projectId: z.string(),
});
export const validateNumericRangeFields = (
data: Pick<ScoreConfigDomain, "maxValue" | "minValue" | "dataType">,
ctx: z.RefinementCtx,
): void | Promise<void> => {
if (data.dataType === "NUMERIC") {
if (
isPresent(data.maxValue) &&
isPresent(data.minValue) &&
data.maxValue <= data.minValue
) {
ctx.addIssue({
code: "custom",
message: "Maximum value must be greater than Minimum value",
});
}
}
};
export const ScoreConfigSchema = z
.discriminatedUnion("dataType", [
z.object({
...ScoreConfigBase.shape,
...NumericConfigFields.shape,
}),
z.object({
...ScoreConfigBase.shape,
...CategoricalConfigFields.shape,
}),
z.object({
...ScoreConfigBase.shape,
...BooleanConfigFields.shape,
}),
z.object({
...ScoreConfigBase.shape,
...TextConfigFields.shape,
}),
])
.superRefine(validateNumericRangeFields);
export type ScoreConfigDomain = z.infer<typeof ScoreConfigSchema>;
export type ScoreConfigCategoryDomain = z.infer<typeof ScoreConfigCategory>;