Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

2.7 KB
Raw Blame
import { ValidateFunction } from "ajv";
import {
  createAjvInstanceInternal,
  FieldValidationError,
  FieldValidationResult,
  validateWithCompiledSchema,
} from "../../../utils/jsonSchemaValidation";

/**
 * Operation-scoped validator that compiles schemas once and reuses them
 * Provides 3800x+ performance improvement over fresh compilation per validation
 * No memory leaks: instance is scoped to the operation and garbage collected after
 *
 * Usage:
 * ```typescript
 * const validator = new DatasetSchemaValidator({ inputSchema, expectedOutputSchema });
 * for (const item of items) {
 *   const result = validator.validateItem(item.input, item.expectedOutput);
 * }
 * // validator is GC'd after operation completes
 * ```
 */
export class DatasetSchemaValidator {
  private inputValidator?: ValidateFunction;
  private outputValidator?: ValidateFunction;

  constructor(params: {
    inputSchema?: Record<string, unknown> | null;
    expectedOutputSchema?: Record<string, unknown> | null;
  }) {
    const ajv = createAjvInstanceInternal();

    if (params.inputSchema) {
      this.inputValidator = ajv.compile(params.inputSchema);
    }

    if (params.expectedOutputSchema) {
      this.outputValidator = ajv.compile(params.expectedOutputSchema);
    }
  }

  /**
   * Validates input field
   */
  validateInput(data: unknown): FieldValidationResult {
    if (!this.inputValidator) {
      return { isValid: true }; // No schema = always valid
    }
    return validateWithCompiledSchema(data, this.inputValidator);
  }

  /**
   * Validates expectedOutput field
   */
  validateOutput(data: unknown): FieldValidationResult {
    if (!this.outputValidator) {
      return { isValid: true }; // No schema = always valid
    }
    return validateWithCompiledSchema(data, this.outputValidator);
  }

  /**
   * Validates both input and expectedOutput fields
   * Returns combined result
   */
  validateItem(params: { input: unknown; expectedOutput: unknown }):
    | { isValid: true }
    | {
        isValid: false;
        inputErrors?: FieldValidationError[];
        expectedOutputErrors?: FieldValidationError[];
      } {
    const errors: {
      inputErrors?: FieldValidationError[];
      expectedOutputErrors?: FieldValidationError[];
    } = {};

    const inputResult = this.validateInput(params.input);
    if (!inputResult.isValid) {
      errors.inputErrors = inputResult.errors;
    }

    const outputResult = this.validateOutput(params.expectedOutput);
    if (!outputResult.isValid) {
      errors.expectedOutputErrors = outputResult.errors;
    }

    const isValid = !errors.inputErrors && !errors.expectedOutputErrors;
    return isValid ? { isValid: true } : { isValid: false, ...errors };
  }
}