Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

1.7 KB
Raw Blame
import { existsSync, readFileSync } from "node:fs";

import { AGENT_EVAL_DIRECTORY, SOURCE_ROOT, WORLD_EVENTS_PATH } from "./paths.js";
import type { AuthoringTranscriptEntry, AuthoringWorldEvent } from "./protocol.js";

interface AgentEvalResults {
  readonly o11y?: {
    readonly shellCommands?: ReadonlyArray<{
      readonly command: string;
      readonly exitCode?: number;
      readonly success?: boolean;
    }>;
  };
}

export interface AuthoringEvalResult {
  readonly commands: ReadonlyArray<string>;
  readonly transcript: ReadonlyArray<AuthoringTranscriptEntry>;
  readonly worldEvents: ReadonlyArray<AuthoringWorldEvent>;
}

export function subjectDefaultAgentModel(): string {
  const source = readFileSync(
    `${SOURCE_ROOT}/packages/eve/src/shared/default-agent-model.ts`,
    "utf8",
  );
  const model = source.match(/DEFAULT_AGENT_MODEL_ID\s*=\s*["']([^"']+)["']/u)?.[1];
  if (model === undefined) throw new Error("Could not read the subject's default agent model.");
  return model;
}

export function authoringEval(): AuthoringEvalResult {
  const results = readJson<AgentEvalResults>(`${AGENT_EVAL_DIRECTORY}/results.json`);
  return {
    commands: (results.o11y?.shellCommands ?? []).map((entry) => entry.command),
    transcript: readJson<ReadonlyArray<AuthoringTranscriptEntry>>(
      `${AGENT_EVAL_DIRECTORY}/harness-transcript.json`,
    ),
    worldEvents: readJsonLines(WORLD_EVENTS_PATH),
  };
}

function readJson<T>(path: string): T {
  return JSON.parse(readFileSync(path, "utf8")) as T;
}

function readJsonLines(path: string): AuthoringWorldEvent[] {
  if (!existsSync(path)) return [];
  return readFileSync(path, "utf8")
    .split("\n")
    .filter(Boolean)
    .map((line) => JSON.parse(line) as AuthoringWorldEvent);
}