Skip to content

Claude Code. Any Model. The most powerful AI coding agent now speaks every language.

test(madbench): print one variant per line in gen-experiments output

Jack Rudenko3d44107e5d35
diff --git a/ai-docs/benches/recommended-models/experiments/fast.cli.experiment.yaml b/ai-docs/benches/recommended-models/experiments/fast.cli.experiment.yaml
index c65a7f39..172956af 100644
--- a/ai-docs/benches/recommended-models/experiments/fast.cli.experiment.yaml
+++ b/ai-docs/benches/recommended-models/experiments/fast.cli.experiment.yaml
@@ -1,5 +1,5 @@
# GENERATED by scripts/gen-experiments.ts from `claudish --models-top --json`.
-# Catalog generation g-20261014030532231-748cc4dd, written 2026-10-14T03:37:15.986Z.
+# Catalog generation g-20261014030532231-748cc4dd, written 2026-10-14T03:43:21.347Z.
# Do not edit by hand: re-run the script. 8 variants.
description: "cli: recommended fast models"
bench: ../cli.madbench.yaml
diff --git a/ai-docs/benches/recommended-models/experiments/fast.mcp.experiment.yaml b/ai-docs/benches/recommended-models/experiments/fast.mcp.experiment.yaml
index 35b9606d..59747bf3 100644
--- a/ai-docs/benches/recommended-models/experiments/fast.mcp.experiment.yaml
+++ b/ai-docs/benches/recommended-models/experiments/fast.mcp.experiment.yaml
@@ -1,5 +1,5 @@
# GENERATED by scripts/gen-experiments.ts from `claudish --models-top --json`.
-# Catalog generation g-20261014030532231-748cc4dd, written 2026-10-14T03:37:15.986Z.
+# Catalog generation g-20261014030532231-748cc4dd, written 2026-10-14T03:43:21.347Z.
# Do not edit by hand: re-run the script. 8 variants.
description: "mcp: recommended fast models"
bench: ../mcp.madbench.yaml
diff --git a/ai-docs/benches/recommended-models/experiments/programming.cli.experiment.yaml b/ai-docs/benches/recommended-models/experiments/programming.cli.experiment.yaml
index cbf4b422..620fe985 100644
--- a/ai-docs/benches/recommended-models/experiments/programming.cli.experiment.yaml
+++ b/ai-docs/benches/recommended-models/experiments/programming.cli.experiment.yaml
@@ -1,5 +1,5 @@
# GENERATED by scripts/gen-experiments.ts from `claudish --models-top --json`.
-# Catalog generation g-20261014030532231-748cc4dd, written 2026-10-14T03:37:15.986Z.
+# Catalog generation g-20261014030532231-748cc4dd, written 2026-10-14T03:43:21.347Z.
# Do not edit by hand: re-run the script. 9 variants.
description: "cli: recommended programming models"
bench: ../cli.madbench.yaml
diff --git a/ai-docs/benches/recommended-models/experiments/programming.mcp.experiment.yaml b/ai-docs/benches/recommended-models/experiments/programming.mcp.experiment.yaml
index e8843690..0165340d 100644
--- a/ai-docs/benches/recommended-models/experiments/programming.mcp.experiment.yaml
+++ b/ai-docs/benches/recommended-models/experiments/programming.mcp.experiment.yaml
@@ -1,5 +1,5 @@
# GENERATED by scripts/gen-experiments.ts from `claudish --models-top --json`.
-# Catalog generation g-20261014030532231-748cc4dd, written 2026-10-14T03:37:15.986Z.
+# Catalog generation g-20261014030532231-748cc4dd, written 2026-10-14T03:43:21.347Z.
# Do not edit by hand: re-run the script. 9 variants.
description: "mcp: recommended programming models"
bench: ../mcp.madbench.yaml
diff --git a/ai-docs/benches/recommended-models/experiments/subscription.cli.experiment.yaml b/ai-docs/benches/recommended-models/experiments/subscription.cli.experiment.yaml
index 1c2f81bf..47f16384 100644
--- a/ai-docs/benches/recommended-models/experiments/subscription.cli.experiment.yaml
+++ b/ai-docs/benches/recommended-models/experiments/subscription.cli.experiment.yaml
@@ -1,5 +1,5 @@
# GENERATED by scripts/gen-experiments.ts from `claudish --models-top --json`.
-# Catalog generation g-20261014030532231-748cc4dd, written 2026-10-14T03:37:15.986Z.
+# Catalog generation g-20261014030532231-748cc4dd, written 2026-10-14T03:43:21.347Z.
# Do not edit by hand: re-run the script. 21 variants.
description: "cli: recommended subscription models"
bench: ../cli.madbench.yaml
diff --git a/ai-docs/benches/recommended-models/experiments/subscription.mcp.experiment.yaml b/ai-docs/benches/recommended-models/experiments/subscription.mcp.experiment.yaml
index 2595bb40..805099a4 100644
--- a/ai-docs/benches/recommended-models/experiments/subscription.mcp.experiment.yaml
+++ b/ai-docs/benches/recommended-models/experiments/subscription.mcp.experiment.yaml
@@ -1,5 +1,5 @@
# GENERATED by scripts/gen-experiments.ts from `claudish --models-top --json`.
-# Catalog generation g-20261014030532231-748cc4dd, written 2026-10-14T03:37:15.986Z.
+# Catalog generation g-20261014030532231-748cc4dd, written 2026-10-14T03:43:21.347Z.
# Do not edit by hand: re-run the script. 21 variants.
description: "mcp: recommended subscription models"
bench: ../mcp.madbench.yaml
diff --git a/ai-docs/benches/recommended-models/scripts/gen-experiments.ts b/ai-docs/benches/recommended-models/scripts/gen-experiments.ts
index f709d730..b515c964 100644
--- a/ai-docs/benches/recommended-models/scripts/gen-experiments.ts
+++ b/ai-docs/benches/recommended-models/scripts/gen-experiments.ts
@@ -57,6 +57,8 @@ mkdirSync(OUT_DIR, { recursive: true });
for (const f of readdirSync(OUT_DIR)) if (f.endsWith(".experiment.yaml")) rmSync(join(OUT_DIR, f));
const written = new Date().toISOString();
+const files: string[] = [];
+console.log(`Recommended list, catalog generation ${doc.generationId}\n`);
for (const [category, specs] of byCategory) {
for (const bench of ["cli", "mcp"]) {
const lines = [
@@ -78,8 +80,13 @@ for (const [category, specs] of byCategory) {
for (const spec of specs) {
lines.push(` - name: ${JSON.stringify(spec)}`, ` params: { model: ${JSON.stringify(spec)} }`);
}
- writeFileSync(join(OUT_DIR, `${category}.${bench}.experiment.yaml`), `${lines.join("\n")}\n`);
+ const file = `${category}.${bench}.experiment.yaml`;
+ writeFileSync(join(OUT_DIR, file), `${lines.join("\n")}\n`);
+ files.push(`experiments/${file}`);
}
- console.log(`${category}: ${specs.length} variants ${specs.join(" ")}`);
+ console.log(`${category} (${specs.length})`);
+ specs.forEach((spec, i) => console.log(` ${String(i + 1).padStart(2)}. ${spec}`));
+ console.log("");
}
-console.log(`generation ${doc.generationId}`);
+console.log(`Wrote ${files.length} files:`);
+for (const f of files) console.log(` ${f}`);