Skip to content

armgit / kiln

Public Imported from GitHub
0

Build and revise procedural 3D assets with your coding agent. Local MCP server, CLI and TypeScript engine with rendered review, editable source and GLB export.

feat(kiln-studio): incubate authenticated Kiln + Bench generate/rank app (phase 1)

Matt Kissinger606fe1bc82cd
diff --git a/arena/adaptive.ts b/arena/adaptive.ts
new file mode 100644
index 00000000..ffc8cb64
--- /dev/null
+++ b/arena/adaptive.ts
@@ -0,0 +1,121 @@
+/**
+ * Active sampling for the Kiln Arena.
+ *
+ * Given the candidate items and the votes collected so far, pick the *next* pair
+ * to show a human so each vote is maximally informative. The heuristic:
+ *
+ * 1. Prefer pairs we've never compared (or compared least) — coverage.
+ * 2. Among those, prefer pairs whose Bradley-Terry strengths are *closest*
+ * (the outcome is genuinely uncertain, so the vote is most informative).
+ * 3. Boost pairs where the combined uncertainty (std err) is high — items we
+ * haven't pinned down yet.
+ *
+ * Pure + deterministic given the same inputs. No I/O.
+ */
+import { fitBradleyTerry, type Vote, type ItemStrength } from './bradley-terry';
+
+export interface Pair {
+ a: string;
+ b: string;
+}
+
+export interface PickNextPairOptions {
+ /**
+ * Weight on closeness of strengths (default 1). Higher -> prefer "toss-up"
+ * matchups more aggressively.
+ */
+ closenessWeight?: number;
+ /** Weight on combined std-err / uncertainty (default 1). */
+ uncertaintyWeight?: number;
+ /**
+ * Weight discouraging repeat comparisons (default 4). Each prior comparison
+ * of a pair subtracts this much from its score.
+ */
+ repeatPenalty?: number;
+}
+
+/**
+ * Select the most informative next pair to vote on. Returns `null` if fewer than
+ * two items exist.
+ */
+export function pickNextPair(
+ items: readonly string[],
+ votes: readonly Vote[],
+ options: PickNextPairOptions = {},
+): Pair | null {
+ const closenessWeight = options.closenessWeight ?? 1;
+ const uncertaintyWeight = options.uncertaintyWeight ?? 1;
+ const repeatPenalty = options.repeatPenalty ?? 4;
+
+ const ids = [...new Set(items)].sort();
+ if (ids.length < 2) return null;
+
+ // Comparison counts per unordered pair, keyed canonically.
+ const pairCounts = new Map<string, number>();
+ for (const v of votes) {
+ const key = pairKey(v.winner, v.loser);
+ pairCounts.set(key, (pairCounts.get(key) ?? 0) + 1);
+ }
+
+ // Current strength estimates (seed all ids so brand-new items appear).
+ const fit = fitBradleyTerry(votes, { ids });
+ const byId = new Map<string, ItemStrength>();
+ for (const it of fit.items) byId.set(it.id, it);
+
+ // Normalize uncertainty: missing/NaN std err -> treat as maximally uncertain.
+ let maxStdErr = 0;
+ for (const it of fit.items) {
+ if (Number.isFinite(it.stdErr) && it.stdErr > maxStdErr) maxStdErr = it.stdErr;
+ }
+ const stdErrOf = (id: string): number => {
+ const it = byId.get(id);
+ if (!it || !Number.isFinite(it.stdErr)) return maxStdErr > 0 ? maxStdErr : 1;
+ return it.stdErr;
+ };
+ const scoreOf = (id: string): number => byId.get(id)?.score ?? 0;
+
+ // Scale closeness against the observed score spread so weights are comparable.
+ let minScore = Infinity;
+ let maxScore = -Infinity;
+ for (const it of fit.items) {
+ if (it.score < minScore) minScore = it.score;
+ if (it.score > maxScore) maxScore = it.score;
+ }
+ const scoreSpread = maxScore - minScore || 1;
+ const stdErrScale = maxStdErr > 0 ? maxStdErr : 1;
+
+ let best: Pair | null = null;
+ let bestScore = -Infinity;
+
+ // Iterate all unordered pairs deterministically (i < j over sorted ids).
+ for (let i = 0; i < ids.length; i++) {
+ for (let j = i + 1; j < ids.length; j++) {
+ const a = ids[i]!;
+ const b = ids[j]!;
+ const seen = pairCounts.get(pairKey(a, b)) ?? 0;
+
+ // Closeness in [0,1]: 1 when strengths are identical.
+ const closeness = 1 - Math.min(1, Math.abs(scoreOf(a) - scoreOf(b)) / scoreSpread);
+ // Combined uncertainty in [0,~2].
+ const uncertainty = (stdErrOf(a) + stdErrOf(b)) / stdErrScale;
+
+ const informativeness =
+ closenessWeight * closeness +
+ uncertaintyWeight * uncertainty -
+ repeatPenalty * seen;
+
+ // Tie-break deterministically: earlier (a,b) in sorted order wins.
+ if (informativeness > bestScore) {
+ bestScore = informativeness;
+ best = { a, b };
+ }
+ }
+ }
+
+ return best;
+}
+
+/** Canonical unordered-pair key. */
+function pairKey(x: string, y: string): string {
+ return x < y ? `${x} ${y}` : `${y} ${x}`;
+}
diff --git a/arena/arena.test.ts b/arena/arena.test.ts
new file mode 100644
index 00000000..dc3362b2
--- /dev/null
+++ b/arena/arena.test.ts
@@ -0,0 +1,113 @@
+/**
+ * Tests for the pure Kiln Arena ranking math — Bradley-Terry strength fitting +
+ * adaptive pair sampling. Relocated with the implementation from kiln-bench so
+ * the math is regression-covered in core (the single source of truth).
+ */
+import { test, expect } from 'bun:test';
+import { fitBradleyTerry, type Vote } from './bradley-terry';
+import { pickNextPair } from './adaptive';
+
+test('bradley-terry recovers a clear ordering from synthetic votes', () => {
+ // Ground truth strength: a > b > c. Generate votes consistent with that.
+ const votes: Vote[] = [];
+ const win = (w: string, l: string, n: number) => {
+ for (let i = 0; i < n; i++) votes.push({ winner: w, loser: l });
+ };
+ // a dominates b and c; b beats c. A few upsets keep it well-conditioned.
+ win('a', 'b', 8);
+ win('b', 'a', 2);
+ win('a', 'c', 9);
+ win('c', 'a', 1);
+ win('b', 'c', 7);
+ win('c', 'b', 3);
+
+ const result = fitBradleyTerry(votes);
+ const order = result.items.map((i) => i.id);
+
+ expect(order).toEqual(['a', 'b', 'c']);
+ expect(result.converged).toBe(true);
+
+ // Elo scores should be strictly decreasing and finite.
+ const [a, b, c] = result.items;
+ expect(a!.score).toBeGreaterThan(b!.score);
+ expect(b!.score).toBeGreaterThan(c!.score);
+ for (const it of result.items) {
+ expect(Number.isFinite(it.score)).toBe(true);
+ expect(Number.isFinite(it.stdErr)).toBe(true);
+ expect(it.stdErr).toBeGreaterThan(0);
+ }
+});
+
+test('bradley-terry is deterministic', () => {
+ const votes: Vote[] = [
+ { winner: 'x', loser: 'y' },
+ { winner: 'x', loser: 'z' },
+ { winner: 'y', loser: 'z' },
+ ];
+ const r1 = fitBradleyTerry(votes);
+ const r2 = fitBradleyTerry(votes);
+ expect(r1.items.map((i) => [i.id, i.score])).toEqual(r2.items.map((i) => [i.id, i.score]));
+});
+
+test('bradley-terry handles undefeated/winless extremes without NaN strengths', () => {
+ // p beats everyone, r loses to everyone.
+ const votes: Vote[] = [
+ { winner: 'p', loser: 'q' },
+ { winner: 'p', loser: 'r' },
+ { winner: 'q', loser: 'r' },
+ ];
+ const result = fitBradleyTerry(votes);
+ const order = result.items.map((i) => i.id);
+ expect(order).toEqual(['p', 'q', 'r']);
+ for (const it of result.items) {
+ expect(Number.isNaN(it.strength)).toBe(false);
+ expect(it.strength).toBeGreaterThan(0);
+ }
+});
+
+test('bradley-terry seeds unvoted ids at neutral strength', () => {
+ const result = fitBradleyTerry([{ winner: 'a', loser: 'b' }], { ids: ['a', 'b', 'lonely'] });
+ const lonely = result.items.find((i) => i.id === 'lonely');
+ expect(lonely).toBeDefined();
+ expect(lonely!.games).toBe(0);
+ expect(lonely!.strength).toBeGreaterThan(0);
+ expect(Number.isNaN(lonely!.stdErr)).toBe(true); // no info -> not estimable
+});
+
+test('pickNextPair returns null for fewer than two items', () => {
+ expect(pickNextPair([], [])).toBeNull();
+ expect(pickNextPair(['only'], [])).toBeNull();
+});
+
+test('pickNextPair prefers an uncompared pair over a heavily-compared one', () => {
+ const items = ['a', 'b', 'c'];
+ // a vs b compared many times; a vs c and b vs c never compared.
+ const votes: Vote[] = [];
+ for (let i = 0; i < 10; i++) votes.push({ winner: 'a', loser: 'b' });
+
+ const pair = pickNextPair(items, votes);
+ expect(pair).not.toBeNull();
+ const key = [pair!.a, pair!.b].sort().join(' ');
+ expect(key).not.toBe('a b'); // must avoid the saturated pair
+});
+
+test('pickNextPair returns a valid, ordered, distinct pair', () => {
+ const items = ['m', 'n', 'o', 'p'];
+ const pair = pickNextPair(items, [{ winner: 'm', loser: 'n' }])!;
+ expect(pair.a).not.toBe(pair.b);
+ expect(items).toContain(pair.a);
+ expect(items).toContain(pair.b);
+ // canonical ordering (a < b over sorted ids)
+ expect(pair.a < pair.b).toBe(true);
+});
+
+test('pickNextPair is deterministic given identical inputs', () => {
+ const items = ['a', 'b', 'c', 'd'];
+ const votes: Vote[] = [
+ { winner: 'a', loser: 'b' },
+ { winner: 'c', loser: 'd' },
+ ];
+ const p1 = pickNextPair(items, votes);
+ const p2 = pickNextPair(items, votes);
+ expect(p1).toEqual(p2);
+});
diff --git a/arena/bradley-terry.ts b/arena/bradley-terry.ts
new file mode 100644
index 00000000..489c81c2
--- /dev/null
+++ b/arena/bradley-terry.ts
@@ -0,0 +1,220 @@
+/**
+ * Bradley-Terry model — fit latent "strengths" from pairwise win/loss votes.
+ *
+ * Given votes of the form {winner, loser}, we fit a strength parameter pi_i per
+ * item such that P(i beats j) = pi_i / (pi_i + pi_j). We solve for the MLE with
+ * the classic Zermelo iterative algorithm (a minorization-maximization update):
+ *
+ * pi_i <- W_i / sum_{j != i} ( n_ij / (pi_i + pi_j) )
+ *
+ * where W_i is the number of wins for item i and n_ij is the number of
+ * comparisons between i and j. This is deterministic, has no learning rate, and
+ * converges monotonically for connected comparison graphs.
+ *
+ * We report each item's strength, an Elo-scaled score (so numbers are familiar
+ * to humans), and an approximate standard error derived from the diagonal of the
+ * inverse Fisher information of the log-strength parameters.
+ *
+ * Pure + deterministic: same votes in -> same ranking out. No I/O.
+ *
+ * Relocated from packages/kiln-bench/src/arena (2026-06) so the bench and the
+ * Kiln Studio app single-source the same ranking math.
+ */
+
+/** A single pairwise human judgement: `winner` was preferred over `loser`. */
+export interface Vote {
+ winner: string;
+ loser: string;
+}
+
+/** Per-item result, sorted strongest-first by the ranking() helper. */
+export interface ItemStrength {
+ id: string;
+ /** Raw Bradley-Terry strength (positive, geometric-mean-normalized to 1). */
+ strength: number;
+ /** Elo-scaled score (400 / ln(10) * log(strength), centered on `eloBase`). */
+ score: number;
+ /** Approximate standard error of the Elo score (NaN if not estimable). */
+ stdErr: number;
+ /** Total comparisons this item participated in. */
+ games: number;
+ /** Wins for this item. */
+ wins: number;
+}
+
+export interface BradleyTerryResult {
+ items: ItemStrength[];
+ /** Number of MM iterations actually run. */
+ iterations: number;
+ /** True if the update converged within tolerance before the cap. */
+ converged: boolean;
+}
+
+export interface BradleyTerryOptions {
+ /** Include these ids even if they have no votes (strength stays neutral). */
+ ids?: readonly string[];
+ /** Max minorization-maximization iterations. Default 1000. */
+ maxIterations?: number;
+ /** Convergence tolerance on max log-strength delta. Default 1e-9. */
+ tolerance?: number;
+ /** Elo center for the scaled score. Default 1500. */
+ eloBase?: number;
+}
+
+const ELO_SCALE = 400 / Math.LN10; // ~173.7; converts ln-strength to Elo points.
+
+/**
+ * Fit Bradley-Terry strengths from pairwise votes via the Zermelo MM algorithm.
+ * Returns items sorted strongest-first.
+ */
+export function fitBradleyTerry(
+ votes: readonly Vote[],
+ options: BradleyTerryOptions = {},
+): BradleyTerryResult {
+ const maxIterations = options.maxIterations ?? 1000;
+ const tolerance = options.tolerance ?? 1e-9;
+ const eloBase = options.eloBase ?? 1500;
+
+ // Collect the universe of ids (from votes + any explicitly seeded ids).
+ const idSet = new Set<string>(options.ids ?? []);
+ for (const v of votes) {
+ idSet.add(v.winner);
+ idSet.add(v.loser);
+ }
+ const ids = [...idSet].sort(); // sort -> deterministic indexing
+ const n = ids.length;
+ const index = new Map<string, number>();
+ ids.forEach((id, i) => index.set(id, i));
+
+ // Pairwise comparison counts and per-item win/game tallies.
+ // pairCount[i][j] = number of comparisons between i and j (symmetric).
+ const pairCount: number[][] = Array.from({ length: n }, () => new Array<number>(n).fill(0));
+ const wins = new Array<number>(n).fill(0);
+ const games = new Array<number>(n).fill(0);
+
+ for (const v of votes) {
+ const w = index.get(v.winner);
+ const l = index.get(v.loser);
+ if (w === undefined || l === undefined || w === l) continue;
+ wins[w]! += 1;
+ games[w]! += 1;
+ games[l]! += 1;
+ pairCount[w]![l]! += 1;
+ pairCount[l]![w]! += 1;
+ }
+
+ // Strengths in linear space, start uniform at 1.
+ const pi = new Array<number>(n).fill(1);
+
+ let iterations = 0;
+ let converged = n <= 1; // trivially converged with 0 or 1 item
+
+ for (let iter = 0; iter < maxIterations && !converged; iter++) {
+ iterations = iter + 1;
+ let maxDelta = 0;
+
+ // MM update. Items with no wins cannot get a finite MLE (strength -> 0);
+ // we hold them at a tiny floor so the rest of the system stays well-defined.
+ const next = new Array<number>(n).fill(0);
+ for (let i = 0; i < n; i++) {
+ if (games[i] === 0) {
+ next[i] = pi[i]!; // untouched, stays neutral
+ continue;
+ }
+ let denom = 0;
+ for (let j = 0; j < n; j++) {
+ const nij = pairCount[i]![j]!;
+ if (nij === 0) continue;
+ denom += nij / (pi[i]! + pi[j]!);
+ }
+ // wins[i] may be 0 -> raw MLE strength 0 (a winless item). Floor it to a
+ // tiny positive value so log-space math stays finite and the item still
+ // ranks last rather than blowing up to -Infinity Elo.
+ const raw = denom > 0 ? wins[i]! / denom : pi[i]!;
+ next[i] = Math.max(raw, 1e-6);
+ }
+
+ // Normalize to geometric mean 1 over items that actually played, so the
+ // scale is fixed and comparable across runs.
+ normalizeGeometric(next, games);
+
+ for (let i = 0; i < n; i++) {
+ const a = Math.max(pi[i]!, 1e-12);
+ const b = Math.max(next[i]!, 1e-12);
+ const d = Math.abs(Math.log(b) - Math.log(a));
+ if (d > maxDelta) maxDelta = d;
+ pi[i] = next[i]!;
+ }
+
+ if (maxDelta < tolerance) converged = true;
+ }
+
+ const stdErrs = approxStdErr(pi, pairCount, games);
+
+ const items: ItemStrength[] = ids.map((id, i) => {
+ const strength = pi[i]!;
+ const score = eloBase + ELO_SCALE * Math.log(Math.max(strength, 1e-12));
+ return {
+ id,
+ strength,
+ score,
+ stdErr: stdErrs[i]!,
+ games: games[i]!,
+ wins: wins[i]!,
+ };
+ });
+
+ // Strongest first; tie-break by id for determinism.
+ items.sort((a, b) => (b.strength - a.strength) || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+
+ return { items, iterations, converged };
+}
+
+/** Normalize strengths so the geometric mean over *played* items equals 1. */
+function normalizeGeometric(pi: number[], games: readonly number[]): void {
+ let logSum = 0;
+ let count = 0;
+ for (let i = 0; i < pi.length; i++) {
+ if (games[i]! > 0 && pi[i]! > 0) {
+ logSum += Math.log(pi[i]!);
+ count += 1;
+ }
+ }
+ if (count === 0) return;
+ const factor = Math.exp(-logSum / count);
+ for (let i = 0; i < pi.length; i++) pi[i] = pi[i]! * factor;
+}
+
+/**
+ * Approximate standard error of each item's Elo score.
+ *
+ * Parameterize by beta_i = ln(pi_i). The Fisher information for Bradley-Terry is
+ * I_ii = sum_{j} n_ij * p_ij * (1 - p_ij)
+ * I_ij = -n_ij * p_ij * (1 - p_ij) (i != j)
+ * where p_ij = pi_i / (pi_i + pi_j). The information matrix is rank-deficient by
+ * one (overall scale is unidentifiable), so we invert a per-item 1-D
+ * approximation: Var(beta_i) ~= 1 / I_ii. SE on the Elo scale is ELO_SCALE * SE(beta).
+ *
+ * This is a cheap, well-behaved approximation — good enough to surface "this
+ * item has barely been compared, trust its rank less" in the UI.
+ */
+function approxStdErr(
+ pi: readonly number[],
+ pairCount: readonly number[][],
+ games: readonly number[],
+): number[] {
+ const n = pi.length;
+ const out = new Array<number>(n).fill(NaN);
+ for (let i = 0; i < n; i++) {
+ if (games[i] === 0) continue;
+ let info = 0;
+ for (let j = 0; j < n; j++) {
+ const nij = pairCount[i]![j]!;
+ if (nij === 0) continue;
+ const pij = pi[i]! / (pi[i]! + pi[j]!);
+ info += nij * pij * (1 - pij);
+ }
+ if (info > 0) out[i] = ELO_SCALE * Math.sqrt(1 / info);
+ }
+ return out;
+}
diff --git a/arena/index.ts b/arena/index.ts
new file mode 100644
index 00000000..b418c798
--- /dev/null
+++ b/arena/index.ts
@@ -0,0 +1,21 @@
+/**
+ * `@pixel-forge/core/kiln/arena` — pure, deterministic ranking math for the Kiln
+ * Arena: Bradley-Terry strength fitting + active (adaptive) pair sampling.
+ *
+ * Relocated from packages/kiln-bench/src/arena (2026-06) so both the bench and
+ * the Kiln Studio app single-source the same implementation. Zero dependencies,
+ * no I/O — same votes in, same ranking out.
+ *
+ * - {@link fitBradleyTerry} — fit latent strengths + Elo scores from pairwise votes
+ * - {@link pickNextPair} — choose the most informative next matchup to vote on
+ */
+export { fitBradleyTerry } from './bradley-terry';
+export type {
+ Vote,
+ ItemStrength,
+ BradleyTerryResult,
+ BradleyTerryOptions,
+} from './bradley-terry';
+
+export { pickNextPair } from './adaptive';
+export type { Pair, PickNextPairOptions } from './adaptive';