Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

2.0 KB
Raw Blame
import type { EvalReporter } from "eve/evals/reporters";

const scheduled = new Set<string>();
const completed = new Set<string>();
let targetKind: "local" | "remote" = "remote";

/** Proves the public reporter lifecycle survives the full CLI/runtime boundary. */
export const evalLifecycleReporter: EvalReporter = {
  onRunStart(_evaluations, target) {
    scheduled.clear();
    completed.clear();
    targetKind = target.kind;
  },
  onEvalStart(event) {
    scheduled.add(event.evaluation.id);
  },
  onSessionStart(event) {
    if (!scheduled.has(event.evaluation.id)) {
      throw new Error(`Session callback preceded eval start: ${event.evaluation.id}`);
    }
    assertTraceContext(event.traceContext);
  },
  onEvalComplete(result, context) {
    if (!scheduled.has(result.id)) {
      throw new Error(`Completion callback preceded eval start: ${result.id}`);
    }
    if (context === undefined) {
      throw new Error(`Completion callback omitted context: ${result.id}`);
    }
    for (const traceContext of context.traceContexts) {
      assertTraceContext(traceContext);
    }
    if (targetKind === "local" && context.traceContexts.length === 0) {
      throw new Error(`Local eval exposed no trace context: ${result.id}`);
    }
    completed.add(result.id);
  },
  onRunComplete(summary) {
    if (scheduled.size !== summary.results.length || completed.size !== summary.results.length) {
      throw new Error(
        `Reporter lifecycle mismatch: ${scheduled.size} started, ${completed.size} completed, ${summary.results.length} results.`,
      );
    }
  },
};

function assertTraceContext(trace: {
  readonly spanId: string;
  readonly traceFlags: number;
  readonly traceId: string;
}): void {
  if (!/^[0-9a-f]{32}$/u.test(trace.traceId) || !/^[0-9a-f]{16}$/u.test(trace.spanId)) {
    throw new Error(`Invalid trace context: ${JSON.stringify(trace)}`);
  }
  if (!Number.isInteger(trace.traceFlags)) {
    throw new Error(`Invalid trace flags: ${String(trace.traceFlags)}`);
  }
}