🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { ValidateFunction } from "ajv";
import {
createAjvInstanceInternal,
FieldValidationError,
FieldValidationResult,
validateWithCompiledSchema,
} from "../../../utils/jsonSchemaValidation";
/**
* Operation-scoped validator that compiles schemas once and reuses them
* Provides 3800x+ performance improvement over fresh compilation per validation
* No memory leaks: instance is scoped to the operation and garbage collected after
*
* Usage:
* ```typescript
* const validator = new DatasetSchemaValidator({ inputSchema, expectedOutputSchema });
* for (const item of items) {
* const result = validator.validateItem(item.input, item.expectedOutput);
* }
* // validator is GC'd after operation completes
* ```
*/
export class DatasetSchemaValidator {
private inputValidator?: ValidateFunction;
private outputValidator?: ValidateFunction;
constructor(params: {
inputSchema?: Record<string, unknown> | null;
expectedOutputSchema?: Record<string, unknown> | null;
}) {
const ajv = createAjvInstanceInternal();
if (params.inputSchema) {
this.inputValidator = ajv.compile(params.inputSchema);
}
if (params.expectedOutputSchema) {
this.outputValidator = ajv.compile(params.expectedOutputSchema);
}
}
/**
* Validates input field
*/
validateInput(data: unknown): FieldValidationResult {
if (!this.inputValidator) {
return { isValid: true }; // No schema = always valid
}
return validateWithCompiledSchema(data, this.inputValidator);
}
/**
* Validates expectedOutput field
*/
validateOutput(data: unknown): FieldValidationResult {
if (!this.outputValidator) {
return { isValid: true }; // No schema = always valid
}
return validateWithCompiledSchema(data, this.outputValidator);
}
/**
* Validates both input and expectedOutput fields
* Returns combined result
*/
validateItem(params: { input: unknown; expectedOutput: unknown }):
| { isValid: true }
| {
isValid: false;
inputErrors?: FieldValidationError[];
expectedOutputErrors?: FieldValidationError[];
} {
const errors: {
inputErrors?: FieldValidationError[];
expectedOutputErrors?: FieldValidationError[];
} = {};
const inputResult = this.validateInput(params.input);
if (!inputResult.isValid) {
errors.inputErrors = inputResult.errors;
}
const outputResult = this.validateOutput(params.expectedOutput);
if (!outputResult.isValid) {
errors.expectedOutputErrors = outputResult.errors;
}
const isValid = !errors.inputErrors && !errors.expectedOutputErrors;
return isValid ? { isValid: true } : { isValid: false, ...errors };
}
}