Skip to content

Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors

feat(cli): fase 4.3 — comando eval com suites, runs e scorecard ASCII

diegosouzapwb8707dbbb4e0
diff --git a/bin/cli/commands/eval.mjs b/bin/cli/commands/eval.mjs
new file mode 100644
index 0000000000..2027bed021
--- /dev/null
+++ b/bin/cli/commands/eval.mjs
@@ -0,0 +1,268 @@
+import { readFileSync } from "node:fs";
+import { setTimeout as sleep } from "node:timers/promises";
+import { apiFetch } from "../api.mjs";
+import { emit } from "../output.mjs";
+import { t } from "../i18n.mjs";
+
+function truncate(v, len = 30) {
+ if (v == null) return "-";
+ const s = String(v);
+ return s.length > len ? s.slice(0, len - 1) + "…" : s;
+}
+
+function fmtTs(v) {
+ if (!v) return "-";
+ try {
+ return new Date(v).toLocaleString();
+ } catch {
+ return String(v);
+ }
+}
+
+const suiteSchema = [
+ { key: "id", header: "Suite ID", width: 22 },
+ { key: "name", header: "Name", width: 30 },
+ { key: "samples", header: "Samples" },
+ { key: "rubric", header: "Rubric", width: 16 },
+ { key: "updatedAt", header: "Updated", formatter: fmtTs },
+];
+
+const runSchema = [
+ { key: "id", header: "Run ID", width: 22 },
+ { key: "suiteId", header: "Suite", width: 18 },
+ { key: "status", header: "Status", width: 12 },
+ { key: "model", header: "Model", width: 25 },
+ { key: "score", header: "Score", formatter: (v) => (v != null ? v.toFixed(3) : "-") },
+ {
+ key: "duration",
+ header: "Duration",
+ formatter: (v) => (v != null ? `${(v / 1000).toFixed(1)}s` : "-"),
+ },
+ { key: "startedAt", header: "Started", formatter: fmtTs },
+];
+
+const sampleSchema = [
+ { key: "id", header: "Sample", width: 14 },
+ { key: "score", header: "Score", formatter: (v) => (v != null ? v.toFixed(2) : "-") },
+ { key: "passed", header: "✓", formatter: (v) => (v ? "✓" : "✗") },
+ { key: "input", header: "Input", width: 30, formatter: truncate },
+ { key: "output", header: "Output", width: 30, formatter: truncate },
+];
+
+async function confirm(q) {
+ return new Promise((resolve) => {
+ process.stdout.write(`${q} (yes/no) `);
+ process.stdin.setEncoding("utf8");
+ process.stdin.once("data", (c) => resolve(c.toString().trim().toLowerCase().startsWith("y")));
+ });
+}
+
+async function watchRun(runId, globalOpts) {
+ let lastStatus = "";
+ while (true) {
+ await sleep(3000);
+ const res = await apiFetch(`/api/evals/${runId}`);
+ if (!res.ok) continue;
+ const r = await res.json();
+ if (r.status !== lastStatus) {
+ const done = r.progress?.completed ?? 0;
+ const total = r.progress?.total ?? "?";
+ process.stderr.write(`[${new Date().toISOString()}] ${r.status} — ${done}/${total}\n`);
+ lastStatus = r.status;
+ }
+ if (["completed", "failed", "cancelled"].includes(r.status)) {
+ emit(r, globalOpts, runSchema);
+ return;
+ }
+ }
+}
+
+function renderScorecard(data) {
+ const score = data.score ?? data.overallScore ?? null;
+ const passed = data.passed ?? data.summary?.passed ?? null;
+ const total = data.total ?? data.summary?.total ?? null;
+ process.stdout.write("\n=== Scorecard ===\n");
+ if (score != null) process.stdout.write(`Overall score: ${(score * 100).toFixed(1)}%\n`);
+ if (passed != null && total != null) {
+ process.stdout.write(`Passed: ${passed}/${total}\n`);
+ const bar = "█".repeat(Math.round((passed / total) * 20)).padEnd(20, "░");
+ process.stdout.write(`[${bar}] ${((passed / total) * 100).toFixed(0)}%\n`);
+ }
+ const metrics = data.metrics ?? data.breakdown ?? {};
+ for (const [k, v] of Object.entries(metrics)) {
+ process.stdout.write(` ${k}: ${typeof v === "number" ? v.toFixed(3) : v}\n`);
+ }
+ process.stdout.write("\n");
+}
+
+export async function runEvalSuitesList(opts, cmd) {
+ const res = await apiFetch("/api/evals/suites");
+ if (!res.ok) {
+ process.stderr.write(`Error: ${res.status}\n`);
+ process.exit(1);
+ }
+ const data = await res.json();
+ emit(data.items ?? data, cmd.optsWithGlobals(), suiteSchema);
+}
+
+export async function runEvalSuitesGet(id, opts, cmd) {
+ const res = await apiFetch(`/api/evals/suites/${id}`);
+ if (!res.ok) {
+ process.stderr.write(`Not found: ${id}\n`);
+ process.exit(1);
+ }
+ emit(await res.json(), cmd.optsWithGlobals());
+}
+
+export async function runEvalSuitesCreate(opts, cmd) {
+ if (!opts.file) {
+ process.stderr.write("--file required\n");
+ process.exit(2);
+ }
+ const body = JSON.parse(readFileSync(opts.file, "utf8"));
+ const res = await apiFetch("/api/evals/suites", { method: "POST", body });
+ if (!res.ok) {
+ process.stderr.write(`Error: ${res.status}\n`);
+ process.exit(1);
+ }
+ emit(await res.json(), cmd.optsWithGlobals());
+}
+
+export async function runEvalRun(suiteId, opts, cmd) {
+ const globalOpts = cmd.optsWithGlobals();
+ const body = {
+ suiteId,
+ model: opts.model ?? "auto",
+ ...(opts.combo ? { combo: opts.combo } : {}),
+ concurrency: opts.concurrency ?? 4,
+ ...(opts.tag ? { tag: opts.tag } : {}),
+ };
+ const res = await apiFetch("/api/evals", { method: "POST", body });
+ if (!res.ok) {
+ process.stderr.write(`Error: ${res.status}\n`);
+ process.exit(1);
+ }
+ const run = await res.json();
+ emit(run, globalOpts, runSchema);
+ if (opts.watch) {
+ process.stderr.write("\nWatching run... (Ctrl+C to detach)\n");
+ await watchRun(run.id, globalOpts);
+ }
+}
+
+export async function runEvalList(opts, cmd) {
+ const params = new URLSearchParams({ limit: String(opts.limit ?? 50) });
+ if (opts.suite) params.set("suiteId", opts.suite);
+ if (opts.status) params.set("status", opts.status);
+ if (opts.since) params.set("since", opts.since);
+ const res = await apiFetch(`/api/evals?${params}`);
+ if (!res.ok) {
+ process.stderr.write(`Error: ${res.status}\n`);
+ process.exit(1);
+ }
+ const data = await res.json();
+ emit(data.items ?? data, cmd.optsWithGlobals(), runSchema);
+}
+
+export async function runEvalGet(id, opts, cmd) {
+ const res = await apiFetch(`/api/evals/${id}`);
+ if (!res.ok) {
+ process.stderr.write(`Not found: ${id}\n`);
+ process.exit(1);
+ }
+ emit(await res.json(), cmd.optsWithGlobals());
+}
+
+export async function runEvalResults(id, opts, cmd) {
+ const params = new URLSearchParams();
+ if (opts.failed) params.set("filter", "failed");
+ const res = await apiFetch(`/api/evals/${id}?${params}`);
+ if (!res.ok) {
+ process.stderr.write(`Not found: ${id}\n`);
+ process.exit(1);
+ }
+ const data = await res.json();
+ emit(data.samples ?? data.results ?? [], cmd.optsWithGlobals(), sampleSchema);
+}
+
+export async function runEvalCancel(id, opts, cmd) {
+ if (!opts.yes) {
+ const ok = await confirm(`Cancel run ${id}?`);
+ if (!ok) return;
+ }
+ const res = await apiFetch(`/api/evals/${id}`, { method: "POST", body: { op: "cancel" } });
+ if (!res.ok) {
+ process.stderr.write(`Error: ${res.status}\n`);
+ process.exit(1);
+ }
+ process.stdout.write("Cancelled\n");
+}
+
+export async function runEvalScorecard(id, opts, cmd) {
+ const res = await apiFetch(`/api/evals/${id}?scorecard=true`);
+ if (!res.ok) {
+ process.stderr.write(`Not found: ${id}\n`);
+ process.exit(1);
+ }
+ const data = await res.json();
+ const globalOpts = cmd.optsWithGlobals();
+ if (globalOpts.output === "json") {
+ emit(data, globalOpts);
+ } else {
+ renderScorecard(data);
+ }
+}
+
+export function registerEval(program) {
+ const evalCmd = program.command("eval").description(t("eval.description"));
+
+ const suites = evalCmd.command("suites").description(t("eval.suites.description"));
+ suites.command("list").description(t("eval.suites.list.description")).action(runEvalSuitesList);
+ suites
+ .command("get <suiteId>")
+ .description(t("eval.suites.get.description"))
+ .action(runEvalSuitesGet);
+ suites
+ .command("create")
+ .description(t("eval.suites.create.description"))
+ .option("--file <path>", t("eval.suites.create.file"))
+ .action(runEvalSuitesCreate);
+
+ evalCmd
+ .command("run <suiteId>")
+ .description(t("eval.run.description"))
+ .option("-m, --model <id>", t("eval.run.model"), "auto")
+ .option("--combo <name>", t("eval.run.combo"))
+ .option("--concurrency <n>", t("eval.run.concurrency"), parseInt, 4)
+ .option("--tag <tag>", t("eval.run.tag"))
+ .option("--watch", t("eval.run.watch"))
+ .action(runEvalRun);
+
+ evalCmd
+ .command("list")
+ .description(t("eval.list.description"))
+ .option("--suite <id>", t("eval.list.suite"))
+ .option("--status <s>", t("eval.list.status"))
+ .option("--since <ts>", t("eval.list.since"))
+ .option("--limit <n>", t("eval.list.limit"), parseInt, 50)
+ .action(runEvalList);
+
+ evalCmd.command("get <runId>").description(t("eval.get.description")).action(runEvalGet);
+
+ evalCmd
+ .command("results <runId>")
+ .description(t("eval.results.description"))
+ .option("--failed", t("eval.results.failed"))
+ .action(runEvalResults);
+
+ evalCmd
+ .command("cancel <runId>")
+ .description(t("eval.cancel.description"))
+ .option("--yes", t("eval.cancel.yes"))
+ .action(runEvalCancel);
+
+ evalCmd
+ .command("scorecard <runId>")
+ .description(t("eval.scorecard.description"))
+ .action(runEvalScorecard);
+}
diff --git a/tests/unit/cli-eval-commands.test.ts b/tests/unit/cli-eval-commands.test.ts
new file mode 100644
index 0000000000..f623bda3c1
--- /dev/null
+++ b/tests/unit/cli-eval-commands.test.ts
@@ -0,0 +1,219 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+
+const SUITE = {
+ id: "suite-001",
+ name: "Chat quality",
+ samples: 50,
+ rubric: "accuracy",
+ updatedAt: "2026-05-14T10:00:00Z",
+};
+
+const RUN = {
+ id: "run-001",
+ suiteId: "suite-001",
+ status: "completed",
+ model: "gpt-4o",
+ score: 0.87,
+ duration: 42000,
+ startedAt: "2026-05-14T10:00:00Z",
+};
+
+const SAMPLES = [
+ { id: "s1", score: 0.9, passed: true, input: "Hello", output: "Hi there" },
+ { id: "s2", score: 0.4, passed: false, input: "2+2=?", output: "5" },
+];
+
+function makeResp(data: unknown, status = 200) {
+ const obj = {
+ ok: status < 400,
+ status,
+ exitCode: status < 400 ? 0 : 1,
+ json: () => Promise.resolve(data),
+ text: () => Promise.resolve(JSON.stringify(data)),
+ headers: new Headers(),
+ };
+ obj.json = obj.json.bind(obj);
+ obj.text = obj.text.bind(obj);
+ return obj;
+}
+
+async function captureStdout(fn: () => Promise<void>): Promise<string> {
+ const chunks: string[] = [];
+ const orig = process.stdout.write.bind(process.stdout);
+ process.stdout.write = (c: string | Uint8Array) => {
+ chunks.push(typeof c === "string" ? c : c.toString());
+ return true;
+ };
+ try {
+ await fn();
+ } finally {
+ process.stdout.write = orig;
+ }
+ return chunks.join("");
+}
+
+function makeCmd(output = "json") {
+ return { optsWithGlobals: () => ({ output, quiet: output !== "table" }) };
+}
+
+test("runEvalSuitesList retorna lista de suites", async () => {
+ const origFetch = globalThis.fetch;
+ globalThis.fetch = ((url: string) => {
+ assert.ok(url.includes("/api/evals/suites"));
+ return Promise.resolve(makeResp({ items: [SUITE] }));
+ }) as any;
+
+ const { runEvalSuitesList } = await import("../../bin/cli/commands/eval.mjs");
+ const out = await captureStdout(() => runEvalSuitesList({}, makeCmd() as any));
+
+ globalThis.fetch = origFetch;
+ const parsed = JSON.parse(out);
+ assert.ok(Array.isArray(parsed));
+ assert.equal(parsed[0].id, "suite-001");
+});
+
+test("runEvalSuitesGet busca suite por id", async () => {
+ let capturedUrl = "";
+ const origFetch = globalThis.fetch;
+ globalThis.fetch = ((url: string) => {
+ capturedUrl = url;
+ return Promise.resolve(makeResp(SUITE));
+ }) as any;
+
+ const { runEvalSuitesGet } = await import("../../bin/cli/commands/eval.mjs");
+ const out = await captureStdout(() => runEvalSuitesGet("suite-001", {}, makeCmd() as any));
+
+ globalThis.fetch = origFetch;
+ assert.ok(capturedUrl.includes("/api/evals/suites/suite-001"));
+ const parsed = JSON.parse(out);
+ assert.equal(parsed.id, "suite-001");
+});
+
+test("runEvalRun envia suiteId e model no body", async () => {
+ let capturedBody: any = null;
+ let capturedUrl = "";
+ const origFetch = globalThis.fetch;
+ globalThis.fetch = ((url: string, opts: any) => {
+ capturedUrl = url;
+ if (opts?.body) capturedBody = JSON.parse(opts.body);
+ return Promise.resolve(makeResp(RUN));
+ }) as any;
+
+ const { runEvalRun } = await import("../../bin/cli/commands/eval.mjs");
+ await captureStdout(() =>
+ runEvalRun("suite-001", { model: "gpt-4o", concurrency: 4 }, makeCmd() as any)
+ );
+
+ globalThis.fetch = origFetch;
+ assert.ok(capturedUrl.includes("/api/evals"));
+ assert.equal(capturedBody.suiteId, "suite-001");
+ assert.equal(capturedBody.model, "gpt-4o");
+ assert.equal(capturedBody.concurrency, 4);
+});
+
+test("runEvalList envia filtros na query", async () => {
+ let capturedUrl = "";
+ const origFetch = globalThis.fetch;
+ globalThis.fetch = ((url: string) => {
+ capturedUrl = url;
+ return Promise.resolve(makeResp({ items: [RUN] }));
+ }) as any;
+
+ const { runEvalList } = await import("../../bin/cli/commands/eval.mjs");
+ await captureStdout(() =>
+ runEvalList({ suite: "suite-001", status: "completed", limit: 25 }, makeCmd() as any)
+ );
+
+ globalThis.fetch = origFetch;
+ assert.ok(capturedUrl.includes("suiteId=suite-001"));
+ assert.ok(capturedUrl.includes("status=completed"));
+ assert.ok(capturedUrl.includes("limit=25"));
+});
+
+test("runEvalGet busca run por id", async () => {
+ let capturedUrl = "";
+ const origFetch = globalThis.fetch;
+ globalThis.fetch = ((url: string) => {
+ capturedUrl = url;
+ return Promise.resolve(makeResp(RUN));
+ }) as any;
+
+ const { runEvalGet } = await import("../../bin/cli/commands/eval.mjs");
+ const out = await captureStdout(() => runEvalGet("run-001", {}, makeCmd() as any));
+
+ globalThis.fetch = origFetch;
+ assert.ok(capturedUrl.includes("/api/evals/run-001"));
+ const parsed = JSON.parse(out);
+ assert.equal(parsed.id, "run-001");
+});
+
+test("runEvalResults mostra amostras", async () => {
+ const origFetch = globalThis.fetch;
+ globalThis.fetch = ((_url: string) => {
+ return Promise.resolve(makeResp({ samples: SAMPLES }));
+ }) as any;
+
+ const { runEvalResults } = await import("../../bin/cli/commands/eval.mjs");
+ const out = await captureStdout(() => runEvalResults("run-001", {}, makeCmd() as any));
+
+ globalThis.fetch = origFetch;
+ const parsed = JSON.parse(out);
+ assert.ok(Array.isArray(parsed));
+ assert.equal(parsed.length, 2);
+});
+
+test("runEvalResults com --failed envia filter=failed na query", async () => {
+ let capturedUrl = "";
+ const origFetch = globalThis.fetch;
+ globalThis.fetch = ((url: string) => {
+ capturedUrl = url;
+ return Promise.resolve(makeResp({ samples: SAMPLES.filter((s) => !s.passed) }));
+ }) as any;
+
+ const { runEvalResults } = await import("../../bin/cli/commands/eval.mjs");
+ await captureStdout(() => runEvalResults("run-001", { failed: true }, makeCmd() as any));
+
+ globalThis.fetch = origFetch;
+ assert.ok(capturedUrl.includes("filter=failed"));
+});
+
+test("runEvalCancel com --yes envia op: cancel", async () => {
+ let capturedBody: any = null;
+ const origFetch = globalThis.fetch;
+ globalThis.fetch = ((_url: string, opts: any) => {
+ if (opts?.body) capturedBody = JSON.parse(opts.body);
+ return Promise.resolve(makeResp({}));
+ }) as any;
+
+ const out = await captureStdout(async () => {
+ const { runEvalCancel } = await import("../../bin/cli/commands/eval.mjs");
+ await runEvalCancel("run-001", { yes: true }, makeCmd() as any);
+ });
+
+ globalThis.fetch = origFetch;
+ assert.equal(capturedBody.op, "cancel");
+ assert.ok(out.includes("Cancelled"));
+});
+
+test("runEvalScorecard renderiza scorecard em modo table", async () => {
+ const scoreData = {
+ score: 0.87,
+ passed: 43,
+ total: 50,
+ metrics: { accuracy: 0.87, fluency: 0.92 },
+ };
+ const origFetch = globalThis.fetch;
+ globalThis.fetch = ((_url: string) => {
+ return Promise.resolve(makeResp(scoreData));
+ }) as any;
+
+ const { runEvalScorecard } = await import("../../bin/cli/commands/eval.mjs");
+ const out = await captureStdout(() =>
+ runEvalScorecard("run-001", {}, { optsWithGlobals: () => ({ output: "table" }) } as any)
+ );
+
+ globalThis.fetch = origFetch;
+ assert.ok(out.includes("87.0%") || out.includes("Scorecard"));
+ assert.ok(out.includes("43/50") || out.includes("43"));
+});