Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors
TypeScript a2aai-agentsai-gatewayanthropicclaudeclaude-codeclinecodexcopilotcursordeepseekfree-aigeminikimillm-gatewaymcpopenaiopenai-proxyqwentoken-saver
feat(cli): fase 4.3 — comando eval com suites, runs e scorecard ASCII
diegosouzapwb8707dbbb4e0
diff --git a/bin/cli/commands/eval.mjs b/bin/cli/commands/eval.mjsnew file mode 100644index 0000000000..2027bed021--- /dev/null+++ b/bin/cli/commands/eval.mjs@@ -0,0 +1,268 @@+import { readFileSync } from "node:fs";+import { setTimeout as sleep } from "node:timers/promises";+import { apiFetch } from "../api.mjs";+import { emit } from "../output.mjs";+import { t } from "../i18n.mjs";++function truncate(v, len = 30) {+ if (v == null) return "-";+ const s = String(v);+ return s.length > len ? s.slice(0, len - 1) + "…" : s;+}++function fmtTs(v) {+ if (!v) return "-";+ try {+ return new Date(v).toLocaleString();+ } catch {+ return String(v);+ }+}++const suiteSchema = [+ { key: "id", header: "Suite ID", width: 22 },+ { key: "name", header: "Name", width: 30 },+ { key: "samples", header: "Samples" },+ { key: "rubric", header: "Rubric", width: 16 },+ { key: "updatedAt", header: "Updated", formatter: fmtTs },+];++const runSchema = [+ { key: "id", header: "Run ID", width: 22 },+ { key: "suiteId", header: "Suite", width: 18 },+ { key: "status", header: "Status", width: 12 },+ { key: "model", header: "Model", width: 25 },+ { key: "score", header: "Score", formatter: (v) => (v != null ? v.toFixed(3) : "-") },+ {+ key: "duration",+ header: "Duration",+ formatter: (v) => (v != null ? `${(v / 1000).toFixed(1)}s` : "-"),+ },+ { key: "startedAt", header: "Started", formatter: fmtTs },+];++const sampleSchema = [+ { key: "id", header: "Sample", width: 14 },+ { key: "score", header: "Score", formatter: (v) => (v != null ? v.toFixed(2) : "-") },+ { key: "passed", header: "✓", formatter: (v) => (v ? "✓" : "✗") },+ { key: "input", header: "Input", width: 30, formatter: truncate },+ { key: "output", header: "Output", width: 30, formatter: truncate },+];++async function confirm(q) {+ return new Promise((resolve) => {+ process.stdout.write(`${q} (yes/no) `);+ process.stdin.setEncoding("utf8");+ process.stdin.once("data", (c) => resolve(c.toString().trim().toLowerCase().startsWith("y")));+ });+}++async function watchRun(runId, globalOpts) {+ let lastStatus = "";+ while (true) {+ await sleep(3000);+ const res = await apiFetch(`/api/evals/${runId}`);+ if (!res.ok) continue;+ const r = await res.json();+ if (r.status !== lastStatus) {+ const done = r.progress?.completed ?? 0;+ const total = r.progress?.total ?? "?";+ process.stderr.write(`[${new Date().toISOString()}] ${r.status} — ${done}/${total}\n`);+ lastStatus = r.status;+ }+ if (["completed", "failed", "cancelled"].includes(r.status)) {+ emit(r, globalOpts, runSchema);+ return;+ }+ }+}++function renderScorecard(data) {+ const score = data.score ?? data.overallScore ?? null;+ const passed = data.passed ?? data.summary?.passed ?? null;+ const total = data.total ?? data.summary?.total ?? null;+ process.stdout.write("\n=== Scorecard ===\n");+ if (score != null) process.stdout.write(`Overall score: ${(score * 100).toFixed(1)}%\n`);+ if (passed != null && total != null) {+ process.stdout.write(`Passed: ${passed}/${total}\n`);+ const bar = "█".repeat(Math.round((passed / total) * 20)).padEnd(20, "░");+ process.stdout.write(`[${bar}] ${((passed / total) * 100).toFixed(0)}%\n`);+ }+ const metrics = data.metrics ?? data.breakdown ?? {};+ for (const [k, v] of Object.entries(metrics)) {+ process.stdout.write(` ${k}: ${typeof v === "number" ? v.toFixed(3) : v}\n`);+ }+ process.stdout.write("\n");+}++export async function runEvalSuitesList(opts, cmd) {+ const res = await apiFetch("/api/evals/suites");+ if (!res.ok) {+ process.stderr.write(`Error: ${res.status}\n`);+ process.exit(1);+ }+ const data = await res.json();+ emit(data.items ?? data, cmd.optsWithGlobals(), suiteSchema);+}++export async function runEvalSuitesGet(id, opts, cmd) {+ const res = await apiFetch(`/api/evals/suites/${id}`);+ if (!res.ok) {+ process.stderr.write(`Not found: ${id}\n`);+ process.exit(1);+ }+ emit(await res.json(), cmd.optsWithGlobals());+}++export async function runEvalSuitesCreate(opts, cmd) {+ if (!opts.file) {+ process.stderr.write("--file required\n");+ process.exit(2);+ }+ const body = JSON.parse(readFileSync(opts.file, "utf8"));+ const res = await apiFetch("/api/evals/suites", { method: "POST", body });+ if (!res.ok) {+ process.stderr.write(`Error: ${res.status}\n`);+ process.exit(1);+ }+ emit(await res.json(), cmd.optsWithGlobals());+}++export async function runEvalRun(suiteId, opts, cmd) {+ const globalOpts = cmd.optsWithGlobals();+ const body = {+ suiteId,+ model: opts.model ?? "auto",+ ...(opts.combo ? { combo: opts.combo } : {}),+ concurrency: opts.concurrency ?? 4,+ ...(opts.tag ? { tag: opts.tag } : {}),+ };+ const res = await apiFetch("/api/evals", { method: "POST", body });+ if (!res.ok) {+ process.stderr.write(`Error: ${res.status}\n`);+ process.exit(1);+ }+ const run = await res.json();+ emit(run, globalOpts, runSchema);+ if (opts.watch) {+ process.stderr.write("\nWatching run... (Ctrl+C to detach)\n");+ await watchRun(run.id, globalOpts);+ }+}++export async function runEvalList(opts, cmd) {+ const params = new URLSearchParams({ limit: String(opts.limit ?? 50) });+ if (opts.suite) params.set("suiteId", opts.suite);+ if (opts.status) params.set("status", opts.status);+ if (opts.since) params.set("since", opts.since);+ const res = await apiFetch(`/api/evals?${params}`);+ if (!res.ok) {+ process.stderr.write(`Error: ${res.status}\n`);+ process.exit(1);+ }+ const data = await res.json();+ emit(data.items ?? data, cmd.optsWithGlobals(), runSchema);+}++export async function runEvalGet(id, opts, cmd) {+ const res = await apiFetch(`/api/evals/${id}`);+ if (!res.ok) {+ process.stderr.write(`Not found: ${id}\n`);+ process.exit(1);+ }+ emit(await res.json(), cmd.optsWithGlobals());+}++export async function runEvalResults(id, opts, cmd) {+ const params = new URLSearchParams();+ if (opts.failed) params.set("filter", "failed");+ const res = await apiFetch(`/api/evals/${id}?${params}`);+ if (!res.ok) {+ process.stderr.write(`Not found: ${id}\n`);+ process.exit(1);+ }+ const data = await res.json();+ emit(data.samples ?? data.results ?? [], cmd.optsWithGlobals(), sampleSchema);+}++export async function runEvalCancel(id, opts, cmd) {+ if (!opts.yes) {+ const ok = await confirm(`Cancel run ${id}?`);+ if (!ok) return;+ }+ const res = await apiFetch(`/api/evals/${id}`, { method: "POST", body: { op: "cancel" } });+ if (!res.ok) {+ process.stderr.write(`Error: ${res.status}\n`);+ process.exit(1);+ }+ process.stdout.write("Cancelled\n");+}++export async function runEvalScorecard(id, opts, cmd) {+ const res = await apiFetch(`/api/evals/${id}?scorecard=true`);+ if (!res.ok) {+ process.stderr.write(`Not found: ${id}\n`);+ process.exit(1);+ }+ const data = await res.json();+ const globalOpts = cmd.optsWithGlobals();+ if (globalOpts.output === "json") {+ emit(data, globalOpts);+ } else {+ renderScorecard(data);+ }+}++export function registerEval(program) {+ const evalCmd = program.command("eval").description(t("eval.description"));++ const suites = evalCmd.command("suites").description(t("eval.suites.description"));+ suites.command("list").description(t("eval.suites.list.description")).action(runEvalSuitesList);+ suites+ .command("get <suiteId>")+ .description(t("eval.suites.get.description"))+ .action(runEvalSuitesGet);+ suites+ .command("create")+ .description(t("eval.suites.create.description"))+ .option("--file <path>", t("eval.suites.create.file"))+ .action(runEvalSuitesCreate);++ evalCmd+ .command("run <suiteId>")+ .description(t("eval.run.description"))+ .option("-m, --model <id>", t("eval.run.model"), "auto")+ .option("--combo <name>", t("eval.run.combo"))+ .option("--concurrency <n>", t("eval.run.concurrency"), parseInt, 4)+ .option("--tag <tag>", t("eval.run.tag"))+ .option("--watch", t("eval.run.watch"))+ .action(runEvalRun);++ evalCmd+ .command("list")+ .description(t("eval.list.description"))+ .option("--suite <id>", t("eval.list.suite"))+ .option("--status <s>", t("eval.list.status"))+ .option("--since <ts>", t("eval.list.since"))+ .option("--limit <n>", t("eval.list.limit"), parseInt, 50)+ .action(runEvalList);++ evalCmd.command("get <runId>").description(t("eval.get.description")).action(runEvalGet);++ evalCmd+ .command("results <runId>")+ .description(t("eval.results.description"))+ .option("--failed", t("eval.results.failed"))+ .action(runEvalResults);++ evalCmd+ .command("cancel <runId>")+ .description(t("eval.cancel.description"))+ .option("--yes", t("eval.cancel.yes"))+ .action(runEvalCancel);++ evalCmd+ .command("scorecard <runId>")+ .description(t("eval.scorecard.description"))+ .action(runEvalScorecard);+}diff --git a/tests/unit/cli-eval-commands.test.ts b/tests/unit/cli-eval-commands.test.tsnew file mode 100644index 0000000000..f623bda3c1--- /dev/null+++ b/tests/unit/cli-eval-commands.test.ts@@ -0,0 +1,219 @@+import test from "node:test";+import assert from "node:assert/strict";++const SUITE = {+ id: "suite-001",+ name: "Chat quality",+ samples: 50,+ rubric: "accuracy",+ updatedAt: "2026-05-14T10:00:00Z",+};++const RUN = {+ id: "run-001",+ suiteId: "suite-001",+ status: "completed",+ model: "gpt-4o",+ score: 0.87,+ duration: 42000,+ startedAt: "2026-05-14T10:00:00Z",+};++const SAMPLES = [+ { id: "s1", score: 0.9, passed: true, input: "Hello", output: "Hi there" },+ { id: "s2", score: 0.4, passed: false, input: "2+2=?", output: "5" },+];++function makeResp(data: unknown, status = 200) {+ const obj = {+ ok: status < 400,+ status,+ exitCode: status < 400 ? 0 : 1,+ json: () => Promise.resolve(data),+ text: () => Promise.resolve(JSON.stringify(data)),+ headers: new Headers(),+ };+ obj.json = obj.json.bind(obj);+ obj.text = obj.text.bind(obj);+ return obj;+}++async function captureStdout(fn: () => Promise<void>): Promise<string> {+ const chunks: string[] = [];+ const orig = process.stdout.write.bind(process.stdout);+ process.stdout.write = (c: string | Uint8Array) => {+ chunks.push(typeof c === "string" ? c : c.toString());+ return true;+ };+ try {+ await fn();+ } finally {+ process.stdout.write = orig;+ }+ return chunks.join("");+}++function makeCmd(output = "json") {+ return { optsWithGlobals: () => ({ output, quiet: output !== "table" }) };+}++test("runEvalSuitesList retorna lista de suites", async () => {+ const origFetch = globalThis.fetch;+ globalThis.fetch = ((url: string) => {+ assert.ok(url.includes("/api/evals/suites"));+ return Promise.resolve(makeResp({ items: [SUITE] }));+ }) as any;++ const { runEvalSuitesList } = await import("../../bin/cli/commands/eval.mjs");+ const out = await captureStdout(() => runEvalSuitesList({}, makeCmd() as any));++ globalThis.fetch = origFetch;+ const parsed = JSON.parse(out);+ assert.ok(Array.isArray(parsed));+ assert.equal(parsed[0].id, "suite-001");+});++test("runEvalSuitesGet busca suite por id", async () => {+ let capturedUrl = "";+ const origFetch = globalThis.fetch;+ globalThis.fetch = ((url: string) => {+ capturedUrl = url;+ return Promise.resolve(makeResp(SUITE));+ }) as any;++ const { runEvalSuitesGet } = await import("../../bin/cli/commands/eval.mjs");+ const out = await captureStdout(() => runEvalSuitesGet("suite-001", {}, makeCmd() as any));++ globalThis.fetch = origFetch;+ assert.ok(capturedUrl.includes("/api/evals/suites/suite-001"));+ const parsed = JSON.parse(out);+ assert.equal(parsed.id, "suite-001");+});++test("runEvalRun envia suiteId e model no body", async () => {+ let capturedBody: any = null;+ let capturedUrl = "";+ const origFetch = globalThis.fetch;+ globalThis.fetch = ((url: string, opts: any) => {+ capturedUrl = url;+ if (opts?.body) capturedBody = JSON.parse(opts.body);+ return Promise.resolve(makeResp(RUN));+ }) as any;++ const { runEvalRun } = await import("../../bin/cli/commands/eval.mjs");+ await captureStdout(() =>+ runEvalRun("suite-001", { model: "gpt-4o", concurrency: 4 }, makeCmd() as any)+ );++ globalThis.fetch = origFetch;+ assert.ok(capturedUrl.includes("/api/evals"));+ assert.equal(capturedBody.suiteId, "suite-001");+ assert.equal(capturedBody.model, "gpt-4o");+ assert.equal(capturedBody.concurrency, 4);+});++test("runEvalList envia filtros na query", async () => {+ let capturedUrl = "";+ const origFetch = globalThis.fetch;+ globalThis.fetch = ((url: string) => {+ capturedUrl = url;+ return Promise.resolve(makeResp({ items: [RUN] }));+ }) as any;++ const { runEvalList } = await import("../../bin/cli/commands/eval.mjs");+ await captureStdout(() =>+ runEvalList({ suite: "suite-001", status: "completed", limit: 25 }, makeCmd() as any)+ );++ globalThis.fetch = origFetch;+ assert.ok(capturedUrl.includes("suiteId=suite-001"));+ assert.ok(capturedUrl.includes("status=completed"));+ assert.ok(capturedUrl.includes("limit=25"));+});++test("runEvalGet busca run por id", async () => {+ let capturedUrl = "";+ const origFetch = globalThis.fetch;+ globalThis.fetch = ((url: string) => {+ capturedUrl = url;+ return Promise.resolve(makeResp(RUN));+ }) as any;++ const { runEvalGet } = await import("../../bin/cli/commands/eval.mjs");+ const out = await captureStdout(() => runEvalGet("run-001", {}, makeCmd() as any));++ globalThis.fetch = origFetch;+ assert.ok(capturedUrl.includes("/api/evals/run-001"));+ const parsed = JSON.parse(out);+ assert.equal(parsed.id, "run-001");+});++test("runEvalResults mostra amostras", async () => {+ const origFetch = globalThis.fetch;+ globalThis.fetch = ((_url: string) => {+ return Promise.resolve(makeResp({ samples: SAMPLES }));+ }) as any;++ const { runEvalResults } = await import("../../bin/cli/commands/eval.mjs");+ const out = await captureStdout(() => runEvalResults("run-001", {}, makeCmd() as any));++ globalThis.fetch = origFetch;+ const parsed = JSON.parse(out);+ assert.ok(Array.isArray(parsed));+ assert.equal(parsed.length, 2);+});++test("runEvalResults com --failed envia filter=failed na query", async () => {+ let capturedUrl = "";+ const origFetch = globalThis.fetch;+ globalThis.fetch = ((url: string) => {+ capturedUrl = url;+ return Promise.resolve(makeResp({ samples: SAMPLES.filter((s) => !s.passed) }));+ }) as any;++ const { runEvalResults } = await import("../../bin/cli/commands/eval.mjs");+ await captureStdout(() => runEvalResults("run-001", { failed: true }, makeCmd() as any));++ globalThis.fetch = origFetch;+ assert.ok(capturedUrl.includes("filter=failed"));+});++test("runEvalCancel com --yes envia op: cancel", async () => {+ let capturedBody: any = null;+ const origFetch = globalThis.fetch;+ globalThis.fetch = ((_url: string, opts: any) => {+ if (opts?.body) capturedBody = JSON.parse(opts.body);+ return Promise.resolve(makeResp({}));+ }) as any;++ const out = await captureStdout(async () => {+ const { runEvalCancel } = await import("../../bin/cli/commands/eval.mjs");+ await runEvalCancel("run-001", { yes: true }, makeCmd() as any);+ });++ globalThis.fetch = origFetch;+ assert.equal(capturedBody.op, "cancel");+ assert.ok(out.includes("Cancelled"));+});++test("runEvalScorecard renderiza scorecard em modo table", async () => {+ const scoreData = {+ score: 0.87,+ passed: 43,+ total: 50,+ metrics: { accuracy: 0.87, fluency: 0.92 },+ };+ const origFetch = globalThis.fetch;+ globalThis.fetch = ((_url: string) => {+ return Promise.resolve(makeResp(scoreData));+ }) as any;++ const { runEvalScorecard } = await import("../../bin/cli/commands/eval.mjs");+ const out = await captureStdout(() =>+ runEvalScorecard("run-001", {}, { optsWithGlobals: () => ({ output: "table" }) } as any)+ );++ globalThis.fetch = origFetch;+ assert.ok(out.includes("87.0%") || out.includes("Scorecard"));+ assert.ok(out.includes("43/50") || out.includes("43"));+});