Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

35.3 KB
Raw Blame
import { describe, it, expect } from 'vitest';
import {
  ASSESSMENT_INTENTS,
  FIT_VERDICTS,
  classifyFitVerdict,
  classifySampleFailure,
  explainAssessment,
  maxWorkingContextTokens,
  parseParamsBillions,
  rankByIntent,
  scoreAssessment,
  scoreForIntent,
  summarizePerformance,
  buildThroughputReport,
  selectSweepTargets,
  summarizeSweepScopes,
  compareEnvironments,
  describeStaleness,
  measuredFitVerdict,
  reconcileFit,
} from './localModelAssessment.js';

const okSample = (contextTokens, charsPerSecond, ttftMs = 300) => ({
  contextTokens, ok: true, charsPerSecond, ttftMs, totalMs: 1000, chars: 40, error: null,
});
const failSample = (contextTokens, error) => ({
  contextTokens, ok: false, charsPerSecond: null, ttftMs: null, totalMs: null, chars: null, error,
});

describe('classifySampleFailure', () => {
  it('returns null for a sample that succeeded', () => {
    expect(classifySampleFailure(okSample(512, 100))).toBeNull();
  });

  it('distinguishes an architecture refusal from resource exhaustion', () => {
    expect(classifySampleFailure(failSample(512, 'unknown model architecture: mamba2'))).toBe('incompatible');
    expect(classifySampleFailure(failSample(512, 'model requires more system memory (42.0 GiB) than is available'))).toBe('resource');
  });

  it('treats a timeout as resource exhaustion, not incompatibility', () => {
    expect(classifySampleFailure(failSample(16384, 'Timed out after 120000ms'))).toBe('resource');
  });

  it('refuses to guess when the failure carries no error text', () => {
    // A failed sample with no message is NOT evidence of anything — guessing
    // here would produce a permanent `does-not-fit` from a transient blip.
    expect(classifySampleFailure({ ok: false })).toBeNull();
    expect(classifySampleFailure({ ok: false, error: 'connection reset by peer' })).toBeNull();
  });
});

describe('classifyFitVerdict', () => {
  it('is unknown with no samples — never "does not fit"', () => {
    expect(classifyFitVerdict([])).toBe('unknown');
    expect(classifyFitVerdict(undefined)).toBe('unknown');
  });

  it('fits when any context length succeeded, even if a larger one failed', () => {
    const samples = [okSample(512, 120), okSample(4096, 60), failSample(16384, 'out of memory')];
    expect(classifyFitVerdict(samples)).toBe('fits');
  });

  it('reports incompatible ahead of does-not-fit — it is the actionable one', () => {
    const samples = [failSample(512, 'cannot allocate memory'), failSample(4096, 'unsupported model format')];
    expect(classifyFitVerdict(samples)).toBe('incompatible');
  });

  it('reports does-not-fit only when a failure is identifiably about resources', () => {
    expect(classifyFitVerdict([failSample(512, 'not enough memory to load model')])).toBe('does-not-fit');
  });

  it('stays unknown when every run failed for an unattributable reason', () => {
    // A network blip is not evidence about hardware fit. Collapsing it into
    // does-not-fit would permanently mislabel a perfectly good model.
    expect(classifyFitVerdict([failSample(512, 'socket hang up')])).toBe('unknown');
  });

  it('only ever emits documented verdicts', () => {
    const cases = [[], [okSample(1, 1)], [failSample(1, 'oom')], [failSample(1, 'unsupported')], [failSample(1, '?')]];
    for (const samples of cases) expect(FIT_VERDICTS).toContain(classifyFitVerdict(samples));
  });
});

describe('maxWorkingContextTokens', () => {
  it('reports the largest SUCCESSFUL context, ignoring failures at larger sizes', () => {
    expect(maxWorkingContextTokens([okSample(512, 100), okSample(4096, 50), failSample(16384, 'oom')])).toBe(4096);
  });

  it('is null — not 0 — when nothing succeeded', () => {
    expect(maxWorkingContextTokens([failSample(512, 'oom')])).toBeNull();
  });
});

describe('summarizePerformance', () => {
  it('averages only successful samples and reports the degradation trend', () => {
    const perf = summarizePerformance([okSample(512, 200, 100), okSample(4096, 100, 400), failSample(16384, 'oom')]);
    expect(perf.samplesRun).toBe(3);
    expect(perf.samplesOk).toBe(2);
    expect(perf.meanCharsPerSecond).toBe(150);
    expect(perf.meanTtftMs).toBe(250);
    expect(perf.peakCharsPerSecond).toBe(200);
    expect(perf.contextDegradation).toBe(0.5);
  });

  it('measures degradation at the LONGEST context, not at the slowest sample', () => {
    // Results are not monotonic in practice. A mid-range dip must not be
    // reported as the long-context behavior — the number is rendered as
    // "held X% of peak at its longest context", so it has to mean that.
    const perf = summarizePerformance([okSample(512, 100), okSample(4096, 50), okSample(16384, 80)]);
    expect(perf.contextDegradation).toBe(0.8);
  });

  it('clamps degradation at 1 when the longest context measured above the others', () => {
    expect(summarizePerformance([okSample(512, 80), okSample(4096, 100)]).contextDegradation).toBe(1);
  });

  it('does not fake a degradation trend from a single sample', () => {
    // Reporting 1 here would claim "throughput held flat across contexts" on
    // the strength of one data point.
    expect(summarizePerformance([okSample(512, 200)]).contextDegradation).toBeNull();
  });

  it('reports nulls, not zeros, when nothing was measured', () => {
    const perf = summarizePerformance([failSample(512, 'oom')]);
    expect(perf.meanCharsPerSecond).toBeNull();
    expect(perf.meanTtftMs).toBeNull();
    expect(perf.peakCharsPerSecond).toBeNull();
    expect(perf.maxWorkingContextTokens).toBeNull();
    expect(perf.samplesOk).toBe(0);
  });
});

describe('parseParamsBillions', () => {
  it('reads a size from the params field or the model id', () => {
    expect(parseParamsBillions({ params: '32B' })).toBe(32);
    expect(parseParamsBillions({ modelId: 'example-model:14b' })).toBe(14);
    expect(parseParamsBillions({ modelId: 'example-model:1.5b' })).toBe(1.5);
  });

  it('is null when no size marker exists — never defaulted to a small model', () => {
    expect(parseParamsBillions({ modelId: 'example-model:latest' })).toBeNull();
    expect(parseParamsBillions({})).toBeNull();
  });
});

describe('scoreAssessment', () => {
  const base = {
    modelId: 'example-model:14b',
    performance: { meanCharsPerSecond: 120, contextDegradation: 0.8 },
    residentGb: 10,
    environment: { memoryBudgetGb: 40 },
  };

  it('scores every axis that has evidence', () => {
    const scores = scoreAssessment(base);
    expect(scores.capability).toBeGreaterThan(0);
    expect(scores.speed).toBeCloseTo(0.5, 5);
    expect(scores.fidelity).toBeCloseTo(0.8, 5);
    expect(scores.memory).toBeCloseTo(0.75, 5);
  });

  it('caps speed at the interactive anchor rather than rewarding runaway numbers', () => {
    expect(scoreAssessment({ ...base, performance: { meanCharsPerSecond: 5000 } }).speed).toBe(1);
  });

  it('leaves an unmeasured axis null instead of scoring it 0', () => {
    const scores = scoreAssessment({ modelId: 'example-model:latest', performance: {}, environment: {} });
    expect(scores.capability).toBeNull();
    expect(scores.speed).toBeNull();
    expect(scores.fidelity).toBeNull();
    expect(scores.memory).toBeNull();
  });

  it('leaves memory null when the budget probe failed, rather than assuming no headroom', () => {
    expect(scoreAssessment({ ...base, environment: { memoryBudgetGb: null } }).memory).toBeNull();
  });
});

describe('scoreForIntent', () => {
  const full = { capability: 0.5, speed: 0.5, fidelity: 0.5, memory: 0.5 };

  it('produces the same score for uniform axes under every intent', () => {
    for (const intent of ASSESSMENT_INTENTS) {
      expect(scoreForIntent(full, intent).score).toBeCloseTo(0.5, 5);
    }
  });

  it('weights the intent it is asked for', () => {
    const fast = { capability: 0, speed: 1, fidelity: 0, memory: 0 };
    expect(scoreForIntent(fast, 'fastest').score).toBeGreaterThan(scoreForIntent(fast, 'smartest').score);
  });

  it('renormalizes over measured axes instead of penalizing missing evidence', () => {
    // Speed-only evidence at 1.0 should score 1.0 — not 0.3 (its balanced
    // weight), which would make a partly-measured model look mediocre.
    const partial = { capability: null, speed: 1, fidelity: null, memory: null };
    const result = scoreForIntent(partial, 'balanced');
    expect(result.score).toBe(1);
    expect(result.coverage).toBe(0.3);
  });

  it('is null — not 0 — when no axis was measured', () => {
    expect(scoreForIntent({ capability: null, speed: null, fidelity: null, memory: null }, 'balanced'))
      .toEqual({ score: null, coverage: 0 });
  });

  it('falls back to balanced weights for an unrecognized intent', () => {
    expect(scoreForIntent(full, 'nonsense').score).toBeCloseTo(0.5, 5);
  });
});

describe('explainAssessment', () => {
  it('names only measured numbers', () => {
    const text = explainAssessment({
      performance: { meanCharsPerSecond: 120, maxWorkingContextTokens: 4096, contextDegradation: 0.8 },
      residentGb: 10,
    }, 'balanced');
    expect(text).toContain('120 chars/s measured');
    expect(text).toContain('4,096 tokens');
    expect(text).toContain('80% of peak');
    expect(text).toContain('10 GB resident');
  });

  it('omits an unmeasured axis rather than describing it', () => {
    const text = explainAssessment({ performance: { meanCharsPerSecond: 120 }, residentGb: null }, 'fastest');
    expect(text).toBe('120 chars/s measured.');
  });

  it('says so plainly when nothing was measured', () => {
    expect(explainAssessment({ performance: {} }, 'smartest')).toMatch(/No measurements recorded/);
  });
});

// The tuning IS the row's identity when a model holds several measurements. The
// consumer keys the row on it, deletes THIS measurement with it, and pre-fills a
// re-measure from it — so a projection that drops it collapses every variant
// onto the backend-defaults record (wrong row deleted, re-measure loses its
// settings, duplicate React keys).
describe('rankByIntent — tuning identity', () => {
  const tuned = (tuningKey, tuning, tuningLabel, charsPerSecond) => ({
    backend: 'llama',
    modelId: 'example-7b',
    verdict: 'fits',
    params: '7B',
    tuningKey,
    tuning,
    tuningLabel,
    performance: { meanCharsPerSecond: charsPerSecond, contextDegradation: 0.9, maxWorkingContextTokens: 16384 },
    environment: { memoryBudgetGb: 64 },
    residentGb: 5,
  });

  it('carries the tuning through to every ranked row', () => {
    const { ranked } = rankByIntent([
      tuned('', {}, null, 90),
      tuned('ubatchSize=512', { ubatchSize: 512 }, 'Micro-batch size 512', 120),
    ], 'fastest');
    expect(ranked).toHaveLength(2);
    expect(ranked.map((r) => r.tuningKey).sort()).toEqual(['', 'ubatchSize=512']);
    const fastest = ranked[0];
    expect(fastest.tuningKey).toBe('ubatchSize=512');
    expect(fastest.tuning).toEqual({ ubatchSize: 512 });
    expect(fastest.tuningLabel).toBe('Micro-batch size 512');
  });

  it('gives two tunings of one model distinct identities, not one collapsed row', () => {
    const { ranked } = rankByIntent([
      tuned('', {}, null, 90),
      tuned('ubatchSize=512', { ubatchSize: 512 }, 'Micro-batch size 512', 120),
    ], 'fastest');
    const keys = ranked.map((r) => `${r.backend}:${r.modelId}@${r.tuningKey}`);
    expect(new Set(keys).size).toBe(2);
  });

  it('reports that a tuning was not applied so the row can say the numbers are another config', () => {
    const entry = tuned('ubatchSize=512', { ubatchSize: 512 }, 'Micro-batch size 512', 120);
    const { ranked } = rankByIntent([
      { ...entry, tuningApplied: false, tuningNotApplied: 'llama-server is not running' },
    ], 'fastest');
    expect(ranked[0].tuningApplied).toBe(false);
    expect(ranked[0].tuningNotApplied).toBe('llama-server is not running');
  });

  it('breaks a model-id tie on the tuning so the order is stable across reloads', () => {
    const a = tuned('a=1', {}, 'A', 100);
    const b = tuned('b=2', { }, 'B', 100);
    expect(rankByIntent([b, a], 'fastest').ranked.map((r) => r.tuningKey)).toEqual(['a=1', 'b=2']);
    expect(rankByIntent([a, b], 'fastest').ranked.map((r) => r.tuningKey)).toEqual(['a=1', 'b=2']);
  });

  it('carries the tuning onto an excluded row too, so variants stay distinguishable there', () => {
    const { excluded } = rankByIntent([
      { ...tuned('ubatchSize=512', { ubatchSize: 512 }, 'Micro-batch size 512', 120), verdict: 'does-not-fit' },
    ], 'fastest');
    expect(excluded[0].tuningKey).toBe('ubatchSize=512');
    expect(excluded[0].tuningLabel).toBe('Micro-batch size 512');
  });
});

describe('rankByIntent', () => {
  const assessment = (modelId, verdict, performance, extra = {}) => ({
    backend: 'ollama',
    modelId,
    verdict,
    performance,
    environment: { memoryBudgetGb: 40 },
    ...extra,
  });

  it('ranks a faster model first for "fastest" and a bigger one first for "smartest"', () => {
    const models = [
      assessment('example-small:7b', 'fits', { meanCharsPerSecond: 240, contextDegradation: 1 }, { residentGb: 5 }),
      assessment('example-large:70b', 'fits', { meanCharsPerSecond: 30, contextDegradation: 1 }, { residentGb: 38 }),
    ];
    expect(rankByIntent(models, 'fastest').ranked[0].modelId).toBe('example-small:7b');
    expect(rankByIntent(models, 'smartest').ranked[0].modelId).toBe('example-large:70b');
  });

  it('favors the lightest resident footprint for "lightweight"', () => {
    const models = [
      assessment('example-large:70b', 'fits', { meanCharsPerSecond: 60, contextDegradation: 1 }, { residentGb: 38 }),
      assessment('example-small:7b', 'fits', { meanCharsPerSecond: 60, contextDegradation: 1 }, { residentGb: 5 }),
    ];
    expect(rankByIntent(models, 'lightweight').ranked[0].modelId).toBe('example-small:7b');
  });

  it('never ranks a model that did not run, and explains why it was left out', () => {
    const models = [
      assessment('example-model:70b', 'does-not-fit', { meanCharsPerSecond: null }, { verdictReason: 'out of memory' }),
      assessment('example-model:7b', 'fits', { meanCharsPerSecond: 120, contextDegradation: 1 }, { residentGb: 5 }),
    ];
    const { ranked, excluded } = rankByIntent(models, 'balanced');
    expect(ranked.map((r) => r.modelId)).toEqual(['example-model:7b']);
    expect(excluded).toEqual([
      { backend: 'ollama', modelId: 'example-model:70b', tuningKey: '', tuningLabel: null, verdict: 'does-not-fit', reason: 'out of memory' },
    ]);
  });

  it('excludes a "fits" model whose intent axes were never measured, rather than ranking it last', () => {
    // Scoring it 0 would put a working-but-unmeasured model below a measured-bad
    // one, which is the exact confusion this feature removes.
    const models = [assessment('example-model:latest', 'fits', {})];
    const { ranked, excluded } = rankByIntent(models, 'balanced');
    expect(ranked).toEqual([]);
    expect(excluded[0].reason).toMatch(/no axis of this intent was measured/);
  });

  it('breaks a score tie on evidence coverage, so a fully-measured model wins', () => {
    const measured = assessment('example-b:7b', 'fits', { meanCharsPerSecond: 120, contextDegradation: 1 }, { residentGb: 20 });
    const partial = assessment('example-a:7b', 'fits', { meanCharsPerSecond: 120 });
    const { ranked } = rankByIntent([partial, measured], 'balanced');
    expect(ranked[0].coverage).toBeGreaterThan(ranked[1].coverage);
  });

  it('carries the measured resident size into the ranked entry', () => {
    // The memory axis is scored FROM residentGb, so dropping it from the entry
    // makes a consumer render "not measured" for a value that was measured.
    const models = [assessment('example-model:7b', 'fits', { meanCharsPerSecond: 120, contextDegradation: 1 }, { residentGb: 5, params: '7B' })];
    const { ranked } = rankByIntent(models, 'balanced');
    expect(ranked[0].residentGb).toBe(5);
    expect(ranked[0].params).toBe('7B');
  });

  it('keeps residentGb null when it was never measured', () => {
    const models = [assessment('example-model:7b', 'fits', { meanCharsPerSecond: 120, contextDegradation: 1 })];
    expect(rankByIntent(models, 'balanced').ranked[0].residentGb).toBeNull();
  });

  it('is empty and safe on no input', () => {
    expect(rankByIntent(undefined, 'balanced')).toEqual({ intent: 'balanced', ranked: [], excluded: [] });
  });

  it('falls back to the balanced intent for an unknown one', () => {
    expect(rankByIntent([], 'nonsense').intent).toBe('balanced');
  });
});

describe('compareEnvironments', () => {
  const recorded = { platform: 'darwin', arch: 'arm64', cpuCount: 12, totalMemoryGb: 64, backendVersion: '1.2.3' };

  it('reports current when every comparable field matches', () => {
    expect(compareEnvironments(recorded, { ...recorded })).toEqual({ comparable: true, stale: false, changes: [] });
  });

  it('flags a RAM change — the case the badge exists for', () => {
    const result = compareEnvironments(recorded, { ...recorded, totalMemoryGb: 128 });
    expect(result.stale).toBe(true);
    expect(result.changes).toEqual([{ field: 'totalMemoryGb', label: 'installed memory', from: 64, to: 128 }]);
    expect(describeStaleness(result)).toContain('installed memory 64 → 128');
  });

  it('flags a backend update', () => {
    const result = compareEnvironments(recorded, { ...recorded, backendVersion: '1.3.0' });
    expect(result.stale).toBe(true);
    expect(result.changes[0].field).toBe('backendVersion');
  });

  it('ignores sub-tolerance memory reporting jitter', () => {
    expect(compareEnvironments(recorded, { ...recorded, totalMemoryGb: 64.3 }).stale).toBe(false);
  });

  it('never treats a field absent on either side as a change', () => {
    // Records written before `backendVersion` was captured, and LM Studio (which
    // exposes no version at all), must not read as "the backend changed".
    const legacy = { platform: 'darwin', arch: 'arm64', cpuCount: 12, totalMemoryGb: 64 };
    const result = compareEnvironments(legacy, { ...legacy, backendVersion: '1.3.0' });
    expect(result.stale).toBe(false);
    expect(result.comparable).toBe(true);
  });

  it('reports not-comparable rather than fresh when there is nothing to compare', () => {
    // `comparable: false` is UNKNOWN. A caller must not render it as a clean bill
    // of health — that is the absent-vs-empty sentinel rule.
    expect(compareEnvironments(null, null)).toEqual({ comparable: false, stale: false, changes: [] });
    expect(describeStaleness({ comparable: false, stale: false, changes: [] })).toBeNull();
  });

  it('does not compare transient available memory, which swings constantly', () => {
    const result = compareEnvironments(
      { ...recorded, availableMemoryGb: 40, memoryBudgetGb: 40 },
      { ...recorded, availableMemoryGb: 3, memoryBudgetGb: 3 },
    );
    expect(result.stale).toBe(false);
  });
});

describe('measuredFitVerdict', () => {
  it('reads a comfortable fit when the footprint is well under the budget', () => {
    expect(measuredFitVerdict({
      verdict: 'fits', residentGb: 8, environment: { memoryBudgetGb: 40 },
    })).toBe('comfortable');
  });

  it('reads tight past the same 60% threshold the estimate uses', () => {
    expect(measuredFitVerdict({
      verdict: 'fits', residentGb: 30, environment: { memoryBudgetGb: 40 },
    })).toBe('tight');
  });

  it('says comfortable when it ran but the footprint is unmeasured (LM Studio)', () => {
    expect(measuredFitVerdict({ verdict: 'fits', residentGb: null, environment: {} })).toBe('comfortable');
  });

  it('maps the failure verdicts onto the badge vocabulary', () => {
    expect(measuredFitVerdict({ verdict: 'does-not-fit' })).toBe('too-large');
    expect(measuredFitVerdict({ verdict: 'incompatible' })).toBe('incompatible');
  });

  it('says nothing at all for an unknown verdict or a missing assessment', () => {
    // null ≠ 'unknown': 'unknown' is a badge the ESTIMATE also produces, so
    // collapsing them would make an unmeasured model look measured.
    expect(measuredFitVerdict({ verdict: 'unknown' })).toBeNull();
    expect(measuredFitVerdict(null)).toBeNull();
  });
});

describe('reconcileFit', () => {
  const measurement = { fit: 'too-large', verdict: 'does-not-fit', assessedAt: '2026-01-02T00:00:00.000Z', stale: false };

  it('leaves the estimate alone when nothing was measured', () => {
    const result = reconcileFit('comfortable', null);
    expect(result).toMatchObject({ fit: 'comfortable', fitSource: 'estimated', measuredFit: null, disagrees: false });
  });

  it('prefers the measurement and records the disagreement', () => {
    const result = reconcileFit('comfortable', measurement);
    expect(result).toMatchObject({
      fit: 'too-large',
      fitSource: 'measured',
      estimatedFit: 'comfortable',
      measuredFit: 'too-large',
      disagrees: true,
      verdict: 'does-not-fit',
    });
  });

  it('does not flag a disagreement when both sides agree', () => {
    expect(reconcileFit('too-large', measurement).disagrees).toBe(false);
  });

  it('keeps the estimate when the measurement is stale, but still reports it', () => {
    const result = reconcileFit('comfortable', { ...measurement, stale: true, staleReason: 'RAM changed' });
    expect(result.fit).toBe('comfortable');
    expect(result.fitSource).toBe('estimated');
    expect(result.measuredFit).toBe('too-large');
    expect(result.stale).toBe(true);
    expect(result.staleReason).toBe('RAM changed');
  });

  it('does not claim a disagreement against an unknown estimate', () => {
    expect(reconcileFit('unknown', measurement).disagrees).toBe(false);
    expect(reconcileFit(null, measurement)).toMatchObject({ fit: 'too-large', fitSource: 'measured', disagrees: false });
  });
});

describe('rankByIntent staleness', () => {
  const assessment = (modelId, charsPerSecond, staleness) => ({
    backend: 'ollama',
    modelId,
    verdict: 'fits',
    params: '14B',
    residentGb: 8,
    environment: { memoryBudgetGb: 40 },
    performance: {
      meanCharsPerSecond: charsPerSecond,
      contextDegradation: 0.9,
      maxWorkingContextTokens: 4096,
      samplesRun: 2,
      samplesOk: 2,
    },
    staleness,
  });

  it('never lets a stale reading outrank a fresh one, however good its numbers', () => {
    const { ranked } = rankByIntent([
      assessment('stale-but-fast:14b', 240, { comparable: true, stale: true, changes: [{ field: 'totalMemoryGb', label: 'installed memory', from: 128, to: 32 }] }),
      assessment('fresh-but-slow:14b', 20, { comparable: true, stale: false, changes: [] }),
    ], 'fastest');
    expect(ranked.map((r) => r.modelId)).toEqual(['fresh-but-slow:14b', 'stale-but-fast:14b']);
    // Still listed, though — this panel is where the user re-runs it.
    expect(ranked[1].staleness.stale).toBe(true);
  });

  it('ranks on score alone when nothing is stale', () => {
    const { ranked } = rankByIntent([
      assessment('slow:14b', 20, { comparable: true, stale: false, changes: [] }),
      assessment('fast:14b', 240, { comparable: true, stale: false, changes: [] }),
    ], 'fastest');
    expect(ranked.map((r) => r.modelId)).toEqual(['fast:14b', 'slow:14b']);
  });

  it('treats a record with no staleness annotation as un-demoted', () => {
    // `staleness: null` means the caller never compared — that is not evidence
    // of staleness, so it must not push the entry down the list.
    const { ranked } = rankByIntent([
      assessment('unannotated:14b', 240, undefined),
      assessment('fresh:14b', 20, { comparable: true, stale: false, changes: [] }),
    ], 'fastest');
    expect(ranked[0].modelId).toBe('unannotated:14b');
  });
});

// ---- tokens per second -------------------------------------------------------

const tokenSample = (contextTokens, tokensPerSecond, { estimated = false, promptRate = null } = {}) => ({
  ...okSample(contextTokens, 200),
  tokensPerSecond,
  promptTokensPerSecond: promptRate,
  completionTokens: 96,
  tokensEstimated: estimated,
});

describe('summarizePerformance — token throughput', () => {
  it('averages the measured tokens/s alongside chars/s', () => {
    const perf = summarizePerformance([tokenSample(512, 40), tokenSample(4096, 20)]);
    expect(perf.meanTokensPerSecond).toBe(30);
    expect(perf.peakTokensPerSecond).toBe(40);
    expect(perf.tokensEstimated).toBe(false);
  });

  // A runtime that reports no usage block must not have a tokens/s figure
  // invented from its character count.
  it('reports null token rates when no sample carried one', () => {
    const perf = summarizePerformance([okSample(512, 200), okSample(4096, 100)]);
    expect(perf.meanTokensPerSecond).toBeNull();
    expect(perf.peakTokensPerSecond).toBeNull();
    expect(perf.tokensEstimated).toBeNull();
    expect(perf.meanCharsPerSecond).toBe(150);
  });

  it('marks the mean as estimated when ANY contributing sample was frame-counted', () => {
    const perf = summarizePerformance([tokenSample(512, 40), tokenSample(4096, 20, { estimated: true })]);
    expect(perf.tokensEstimated).toBe(true);
  });
});

describe('explainAssessment — throughput clause', () => {
  it('leads with tokens/s when it was measured', () => {
    const text = explainAssessment({ performance: summarizePerformance([tokenSample(512, 40)]) }, 'fastest');
    expect(text).toContain('40 tok/s');
    expect(text).not.toContain('chars/s');
  });

  it('falls back to chars/s, and never reports both', () => {
    const text = explainAssessment({ performance: summarizePerformance([okSample(512, 200)]) }, 'fastest');
    expect(text).toContain('200 chars/s');
    expect(text).not.toContain('tok/s');
  });
});

describe('buildThroughputReport', () => {
  const record = (modelId, rates) => ({
    backend: 'ollama',
    modelId,
    tuningKey: '',
    verdict: 'fits',
    samples: rates.map(([context, rate]) => tokenSample(context, rate)),
    performance: summarizePerformance(rates.map(([context, rate]) => tokenSample(context, rate))),
  });

  it('sorts fastest first and collects every sampled context as a column', () => {
    const report = buildThroughputReport([
      record('slow-model', [[512, 10], [4096, 8]]),
      record('fast-model', [[512, 60], [4096, 40]]),
    ]);
    expect(report.rows.map((r) => r.modelId)).toEqual(['fast-model', 'slow-model']);
    expect(report.contexts).toEqual([512, 4096]);
    expect(report.rows[0].points.map((p) => p.tokensPerSecond)).toEqual([60, 40]);
    expect(report.rows[0].points.map((p) => p.totalMs)).toEqual([1000, 1000]);
    expect(report.modelsWithTokenRates).toBe(2);
  });

  // Unmeasured is not slow. A runtime that reports no token counts still gets a
  // row — dropping it would make the table look complete when it isn't.
  it('keeps a row with no token rate, and sorts it last rather than as zero', () => {
    const quiet = {
      backend: 'llama',
      modelId: 'quiet-model',
      verdict: 'fits',
      samples: [okSample(512, 300)],
      performance: summarizePerformance([okSample(512, 300)]),
    };
    const report = buildThroughputReport([quiet, record('fast-model', [[512, 5]])]);
    expect(report.rows.map((r) => r.modelId)).toEqual(['fast-model', 'quiet-model']);
    expect(report.rows[1].meanTokensPerSecond).toBeNull();
    expect(report.rows[1].meanCharsPerSecond).toBe(300);
    expect(report.modelsWithTokenRates).toBe(1);
  });

  it('carries a failed sample as failed rather than dropping the context', () => {
    const report = buildThroughputReport([{
      backend: 'ollama',
      modelId: 'big-model',
      verdict: 'fits',
      samples: [tokenSample(512, 12), failSample(16384, 'out of memory')],
      performance: summarizePerformance([tokenSample(512, 12), failSample(16384, 'out of memory')]),
    }]);
    expect(report.contexts).toEqual([512, 16384]);
    const failed = report.rows[0].points.find((p) => p.contextTokens === 16384);
    expect(failed.ok).toBe(false);
    expect(failed.tokensPerSecond).toBeNull();
    expect(failed.error).toBe('out of memory');
  });

  it('returns an empty report for no assessments rather than throwing', () => {
    expect(buildThroughputReport(null)).toEqual({ rows: [], contexts: [], modelsWithTokenRates: 0 });
  });
});

// ---- sweep target selection --------------------------------------------------

describe('selectSweepTargets', () => {
  const stale = {
    backend: 'ollama', modelId: 'stale-model', tuningKey: 'ctx=8192', tuningLabel: '8k context',
    tuning: { numCtx: 8192 }, staleness: { stale: true },
  };
  const fresh = {
    backend: 'ollama', modelId: 'fresh-model', tuningKey: '', tuningLabel: null,
    tuning: {}, staleness: { stale: false },
  };
  const unassessed = [{ backend: 'lmstudio', modelId: 'new-model' }];

  it('queues only never-measured models for the unmeasured scope', () => {
    const targets = selectSweepTargets({ assessments: [stale, fresh], unassessed, scope: 'unmeasured' });
    expect(targets.map((t) => t.modelId)).toEqual(['new-model']);
  });

  it('queues only stale readings for the stale scope', () => {
    const targets = selectSweepTargets({ assessments: [stale, fresh], unassessed, scope: 'stale' });
    expect(targets.map((t) => t.modelId)).toEqual(['stale-model']);
  });

  // A re-measure has to reproduce the configuration that produced the record, or
  // the sweep quietly replaces every tuned reading with a defaults one.
  it('re-uses each record\'s tuning so a re-measure reproduces its configuration', () => {
    const [target] = selectSweepTargets({ assessments: [stale], unassessed: [], scope: 'stale' });
    expect(target.tuning).toEqual({ numCtx: 8192 });
    expect(target.tuningKey).toBe('ctx=8192');
  });

  it('covers unmeasured models FIRST in the all scope, so a cut-short run got the unknowns', () => {
    const targets = selectSweepTargets({ assessments: [stale, fresh], unassessed, scope: 'all' });
    expect(targets.map((t) => t.modelId)).toEqual(['new-model', 'stale-model', 'fresh-model']);
  });

  it('never queues the same model+tuning twice', () => {
    const targets = selectSweepTargets({
      assessments: [fresh, fresh],
      unassessed: [{ backend: 'ollama', modelId: 'fresh-model' }],
      scope: 'all',
    });
    expect(targets).toHaveLength(1);
  });

  it('falls back to the unmeasured scope for an unrecognized scope', () => {
    const targets = selectSweepTargets({ assessments: [stale], unassessed, scope: 'everything-please' });
    expect(targets.map((t) => t.modelId)).toEqual(['new-model']);
  });

  // ---- the second dimension ------------------------------------------------
  // `tunings` is what makes a TUNING sweep a case of this selector rather than a
  // queue of its own: same dedupe, same ordering, same consent-count guarantee.
  const grid = [
    { tuning: {}, key: '', label: null },
    { tuning: { flashAttn: true }, key: 'flashAttn=true', label: 'Flash attention on' },
  ];

  it('restricts the queue to the named model, whatever the scope would have picked', () => {
    const targets = selectSweepTargets({
      assessments: [stale, fresh], unassessed, scope: 'unmeasured',
      only: { backend: 'ollama', modelId: 'fresh-model' },
    });
    expect(targets.map((t) => t.modelId)).toEqual(['fresh-model']);
  });

  it('returns nothing for a model no record or listing names', () => {
    const targets = selectSweepTargets({
      assessments: [stale], unassessed, scope: 'all',
      only: { backend: 'ollama', modelId: 'ghost' },
    });
    expect(targets).toEqual([]);
  });

  it('crosses every selected model with the grid, one measurement per pair', () => {
    const targets = selectSweepTargets({ assessments: [], unassessed, scope: 'unmeasured', tunings: grid });
    expect(targets.map((t) => `${t.modelId}@${t.tuningKey}`))
      .toEqual(['new-model@', 'new-model@flashAttn=true']);
  });

  // The grid REPLACES each record's own tuning — that is the whole point of the
  // dimension. Keeping the stored tuning would measure the same configuration
  // under every variant's label.
  it('replaces a record\'s stored tuning with the grid variant', () => {
    const [baseline, variant] = selectSweepTargets({
      assessments: [stale], unassessed: [], scope: 'stale', tunings: grid,
    });
    expect(baseline.tuning).toEqual({});
    expect(baseline.tuningLabel).toBeNull();
    expect(variant.tuning).toEqual({ flashAttn: true });
    expect(variant.tuningLabel).toBe('Flash attention on');
  });

  // A model that reached the cross under two stored tunings is still ONE model:
  // measuring each variant twice would put the same configuration on both sides
  // of the comparison table.
  it('crosses a model held under several stored tunings only once', () => {
    const other = { ...stale, tuningKey: 'ctx=4096', tuning: { numCtx: 4096 } };
    const targets = selectSweepTargets({
      assessments: [stale, other], unassessed: [], scope: 'stale', tunings: grid,
    });
    expect(targets).toHaveLength(2);
  });

  it('leaves the queue alone for an empty grid rather than emptying it', () => {
    const targets = selectSweepTargets({ assessments: [], unassessed, scope: 'unmeasured', tunings: [] });
    expect(targets.map((t) => t.modelId)).toEqual(['new-model']);
  });

  // ---- embedding models are not assessable ---------------------------------
  // An assessment measures a model by GENERATING with it. Ollama answers every
  // sample aimed at an embedding-only model with
  // `400 "<model>" does not support chat`, so the whole measurement is doomed
  // from the model id alone — and each doomed sample raised an AI-provider
  // investigation task and briefly benched the provider.
  it('never queues an embedding model for the unmeasured scope', () => {
    const targets = selectSweepTargets({
      assessments: [],
      unassessed: [
        { backend: 'ollama', modelId: 'all-minilm:latest' },
        { backend: 'ollama', modelId: 'nomic-embed-text:latest' },
        { backend: 'ollama', modelId: 'qwen3.6:35b' },
      ],
      scope: 'unmeasured',
    });
    expect(targets.map((t) => t.modelId)).toEqual(['qwen3.6:35b']);
  });

  // A bogus record written before the filter existed must not be re-queued
  // forever by the scopes that re-measure stored readings.
  it('never re-queues a stored embedding-model reading', () => {
    const embedRecord = {
      backend: 'ollama', modelId: 'all-minilm:latest', tuningKey: '', tuningLabel: null,
      tuning: {}, staleness: { stale: true },
    };
    expect(selectSweepTargets({ assessments: [embedRecord], unassessed: [], scope: 'stale' })).toEqual([]);
    expect(selectSweepTargets({ assessments: [embedRecord], unassessed: [], scope: 'all' })).toEqual([]);
  });

  it('refuses a named embedding model even though naming one implies the all scope', () => {
    const targets = selectSweepTargets({
      assessments: [], unassessed: [{ backend: 'ollama', modelId: 'all-minilm:latest' }],
      scope: 'unmeasured', only: { backend: 'ollama', modelId: 'all-minilm:latest' },
    });
    expect(targets).toEqual([]);
  });

  it('keeps an embedding model out of the tuning cross too', () => {
    const targets = selectSweepTargets({
      assessments: [], unassessed: [
        { backend: 'ollama', modelId: 'all-minilm:latest' },
        { backend: 'ollama', modelId: 'qwen3.6:35b' },
      ],
      scope: 'unmeasured', tunings: grid,
    });
    expect(targets.map((t) => `${t.modelId}@${t.tuningKey}`))
      .toEqual(['qwen3.6:35b@', 'qwen3.6:35b@flashAttn=true']);
  });
});

describe('summarizeSweepScopes', () => {
  it('counts exactly what each scope would run', () => {
    const counts = summarizeSweepScopes({
      assessments: [{ backend: 'ollama', modelId: 'a', staleness: { stale: true } }],
      unassessed: [{ backend: 'ollama', modelId: 'b' }, { backend: 'ollama', modelId: 'c' }],
    });
    expect(counts).toEqual({ unmeasured: 2, stale: 1, all: 3 });
  });
});