Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

636 B
Raw Blame
import { defineEval } from "eve/evals";

/**
 * Runs `t.score` end to end: one score without a rule, one gated score that
 * decides the eval.
 */
export default defineEval({
  description: "Scores with and without a rule.",

  async test(t) {
    const turn = await t.send('Reply with exactly the text "score ping" and nothing else.');
    t.succeeded();

    const reply = turn.message ?? "";
    t.score({
      score: Math.max(0, 1 - reply.length / 400),
      metadata: { length: reply.length },
    }).label("reply-brevity");
    t.score(reply.includes("score ping") ? 1 : 0)
      .label("mentions-ping")
      .gate();
  },
});