The all-in-one local AI studio for your desktop: chat, image and video generation and a coding agent in one free, open source app. Windows and Linux. No Docker, no terminal, no cloud required.
TypeScript agentai-chatai-studioall-in-onecoding-agentdesktop-appfluximage-generationlinuxllmlocal-ailocal-llmopen-sourceprivacyself-hostedstable-diffusionuncensoreduncensored-aivideo-generationwindows
import { describe, it, expect, beforeEach } from 'vitest'
import {
useBenchmarkStore, getAverageSpeed, getLatestSpeed, getLeaderboard, computeGenerationTps,
getAverageTokens, getAccuracy, getAverageThinkShare, getTruncatedCount, toMarkdownReport,
} from '../benchmarkStore'
import type { BenchmarkResult } from '../../lib/benchmark-prompts'
// ── Helpers ─────────────────────────────────────────────────────
function makeResult(modelName: string, tps: number, overrides: Partial<BenchmarkResult> = {}): BenchmarkResult {
return {
modelName,
promptId: 'speed',
tokensPerSec: tps,
timeToFirstToken: 100,
totalTime: 5000,
totalTokens: tps * 5,
timestamp: Date.now(),
...overrides,
}
}
// ═══════════════════════════════════════════════════════════════
// benchmarkStore
// ═══════════════════════════════════════════════════════════════
describe('benchmarkStore', () => {
beforeEach(() => {
useBenchmarkStore.setState({
results: {},
isRunning: false,
currentModel: null,
currentStep: 0,
totalSteps: 0,
})
})
// ── addResult ──────────────────────────────────────────────
describe('addResult', () => {
it('adds a result for a new model', () => {
useBenchmarkStore.getState().addResult(makeResult('llama3', 45))
const results = useBenchmarkStore.getState().results
expect(results['llama3']).toHaveLength(1)
expect(results['llama3'][0].tokensPerSec).toBe(45)
})
it('accumulates multiple results per model', () => {
useBenchmarkStore.getState().addResult(makeResult('llama3', 40))
useBenchmarkStore.getState().addResult(makeResult('llama3', 50))
useBenchmarkStore.getState().addResult(makeResult('llama3', 45))
expect(useBenchmarkStore.getState().results['llama3']).toHaveLength(3)
})
it('keeps results separate per model', () => {
useBenchmarkStore.getState().addResult(makeResult('llama3', 40))
useBenchmarkStore.getState().addResult(makeResult('mistral', 55))
expect(useBenchmarkStore.getState().results['llama3']).toHaveLength(1)
expect(useBenchmarkStore.getState().results['mistral']).toHaveLength(1)
})
it('creates the array when model has no prior results', () => {
expect(useBenchmarkStore.getState().results['phi3']).toBeUndefined()
useBenchmarkStore.getState().addResult(makeResult('phi3', 60))
expect(useBenchmarkStore.getState().results['phi3']).toBeDefined()
expect(useBenchmarkStore.getState().results['phi3']).toHaveLength(1)
})
it('preserves existing results for other models', () => {
useBenchmarkStore.getState().addResult(makeResult('llama3', 40))
useBenchmarkStore.getState().addResult(makeResult('mistral', 55))
useBenchmarkStore.getState().addResult(makeResult('llama3', 42))
expect(useBenchmarkStore.getState().results['mistral']).toHaveLength(1)
expect(useBenchmarkStore.getState().results['llama3']).toHaveLength(2)
})
})
// ── setRunning ─────────────────────────────────────────────
describe('setRunning', () => {
it('sets running state with model and total', () => {
useBenchmarkStore.getState().setRunning(true, 'llama3', 3)
const state = useBenchmarkStore.getState()
expect(state.isRunning).toBe(true)
expect(state.currentModel).toBe('llama3')
expect(state.totalSteps).toBe(3)
})
it('resets currentStep to 0 when starting', () => {
useBenchmarkStore.setState({ currentStep: 5 })
useBenchmarkStore.getState().setRunning(true, 'llama3', 3)
expect(useBenchmarkStore.getState().currentStep).toBe(0)
})
it('clears model and steps when stopping', () => {
useBenchmarkStore.getState().setRunning(true, 'llama3', 3)
useBenchmarkStore.getState().setRunning(false)
const state = useBenchmarkStore.getState()
expect(state.isRunning).toBe(false)
expect(state.currentModel).toBeNull()
expect(state.totalSteps).toBe(0)
expect(state.currentStep).toBe(0)
})
})
// ── setStep ────────────────────────────────────────────────
describe('setStep', () => {
it('sets currentStep', () => {
useBenchmarkStore.getState().setStep(2)
expect(useBenchmarkStore.getState().currentStep).toBe(2)
})
it('increments through steps', () => {
useBenchmarkStore.getState().setStep(1)
useBenchmarkStore.getState().setStep(2)
useBenchmarkStore.getState().setStep(3)
expect(useBenchmarkStore.getState().currentStep).toBe(3)
})
})
// ── getAverageSpeed ────────────────────────────────────────
describe('getAverageSpeed', () => {
it('returns null for model with no results', () => {
expect(getAverageSpeed({}, 'unknown')).toBeNull()
})
it('returns null for empty results array', () => {
expect(getAverageSpeed({ 'llama3': [] }, 'llama3')).toBeNull()
})
it('returns exact value for single result', () => {
const results = { 'llama3': [makeResult('llama3', 45.3)] }
expect(getAverageSpeed(results, 'llama3')).toBe(45.3)
})
it('calculates average for multiple results', () => {
const results = {
'llama3': [
makeResult('llama3', 40),
makeResult('llama3', 50),
],
}
expect(getAverageSpeed(results, 'llama3')).toBe(45)
})
it('rounds to one decimal place', () => {
const results = {
'llama3': [
makeResult('llama3', 33.33),
makeResult('llama3', 33.33),
makeResult('llama3', 33.34),
],
}
const avg = getAverageSpeed(results, 'llama3')!
// 33.333... rounded to 1 decimal
expect(avg).toBe(33.3)
})
it('returns null for non-existent model key', () => {
const results = { 'llama3': [makeResult('llama3', 40)] }
expect(getAverageSpeed(results, 'nonexistent')).toBeNull()
})
})
// ── getLatestSpeed (Bug W — a user 2026-05-24) ─────
describe('getLatestSpeed', () => {
it('returns null for missing or empty results', () => {
expect(getLatestSpeed({}, 'unknown')).toBeNull()
expect(getLatestSpeed({ llama3: [] }, 'llama3')).toBeNull()
})
it('returns the only result when there is just one', () => {
const r = { llama3: [makeResult('llama3', 42.3, { timestamp: 1_000_000 })] }
expect(getLatestSpeed(r, 'llama3')).toBe(42.3)
})
it('averages prompts within ONE benchmark session (close-together timestamps)', () => {
// Within-session: BENCHMARK_PROMPTS produces multiple addResult calls,
// all within seconds of each other. They should be averaged together.
const t0 = 1_000_000
const r = {
llama3: [
makeResult('llama3', 20, { timestamp: t0 }),
makeResult('llama3', 30, { timestamp: t0 + 2_000 }),
makeResult('llama3', 40, { timestamp: t0 + 4_000 }),
],
}
// (20 + 30 + 40) / 3 = 30
expect(getLatestSpeed(r, 'llama3')).toBe(30)
})
it('IGNORES previous session when a new one starts (>10 s gap)', () => {
// A user scenario: ten Run Benchmark clicks. Old sessions
// should not pollute the displayed number for the most recent click.
const t0 = 1_000_000
const r = {
llama3: [
// Session A: avg 15.2
makeResult('llama3', 14.0, { timestamp: t0 }),
makeResult('llama3', 15.5, { timestamp: t0 + 1_000 }),
makeResult('llama3', 16.1, { timestamp: t0 + 2_000 }),
// 5-minute gap — new session
makeResult('llama3', 17.5, { timestamp: t0 + 300_000 }),
makeResult('llama3', 18.2, { timestamp: t0 + 301_000 }),
makeResult('llama3', 17.9, { timestamp: t0 + 302_000 }),
],
}
// Latest session only: (17.5 + 18.2 + 17.9) / 3 = 17.866... → 17.9
expect(getLatestSpeed(r, 'llama3')).toBe(17.9)
})
it('still averages ALL runs when they are within session gap of each other', () => {
// Edge case: rapid-fire reruns within the 10 s gap stay one session.
const t0 = 1_000_000
const r = {
llama3: [
makeResult('llama3', 10, { timestamp: t0 }),
makeResult('llama3', 20, { timestamp: t0 + 3_000 }),
makeResult('llama3', 30, { timestamp: t0 + 6_000 }),
],
}
expect(getLatestSpeed(r, 'llama3')).toBe(20)
})
})
// ── getLeaderboard ─────────────────────────────────────────
describe('getLeaderboard', () => {
it('returns empty array for empty results', () => {
expect(getLeaderboard({})).toEqual([])
})
it('sorts models by average tokens/sec descending', () => {
const results = {
'slow': [makeResult('slow', 20)],
'fast': [makeResult('fast', 80)],
'medium': [makeResult('medium', 50)],
}
const board = getLeaderboard(results)
expect(board).toHaveLength(3)
expect(board[0].model).toBe('fast')
expect(board[1].model).toBe('medium')
expect(board[2].model).toBe('slow')
})
it('includes correct run counts', () => {
const results = {
'llama3': [makeResult('llama3', 40), makeResult('llama3', 50), makeResult('llama3', 45)],
'mistral': [makeResult('mistral', 55)],
}
const board = getLeaderboard(results)
const llama = board.find(b => b.model === 'llama3')!
const mistral = board.find(b => b.model === 'mistral')!
expect(llama.runs).toBe(3)
expect(mistral.runs).toBe(1)
})
it('calculates correct averages', () => {
const results = {
'llama3': [makeResult('llama3', 40), makeResult('llama3', 50)],
}
const board = getLeaderboard(results)
expect(board[0].avgTps).toBe(45)
})
it('rounds averages to one decimal', () => {
const results = {
'llama3': [makeResult('llama3', 33), makeResult('llama3', 34)],
}
const board = getLeaderboard(results)
expect(board[0].avgTps).toBe(33.5)
})
it('handles single model', () => {
const results = {
'only-one': [makeResult('only-one', 42.7)],
}
const board = getLeaderboard(results)
expect(board).toHaveLength(1)
expect(board[0].model).toBe('only-one')
expect(board[0].avgTps).toBe(42.7)
expect(board[0].runs).toBe(1)
})
})
// ── computeGenerationTps (Bug M — v2.4.7) ──
describe('computeGenerationTps', () => {
it('excludes time-to-first-token from the denominator', () => {
// 100 tokens, 5000ms total, 2000ms TTFT → 3000ms of pure generation
// pre-v2.4.7 formula: 100/5000*1000 = 20 tok/s
// post-v2.4.7 formula: 100/3000*1000 ≈ 33.3 tok/s
const tps = computeGenerationTps(100, 5000, 2000)
expect(tps).toBeCloseTo(33.333, 2)
})
it('matches a user RTX 4070 Laptop scenario', () => {
// nightmare's report: 23-25 tok/s in chat, 12 tok/s in pre-v2.4.7 benchmark.
// Reconstruct: ~60 tokens, total ~5000ms (so 12 tok/s under old math),
// TTFT ~2500ms → generation ~2500ms → 24 tok/s in new math.
const tps = computeGenerationTps(60, 5000, 2500)
expect(tps).toBe(24)
})
it('returns 0 when tokenCount is 0', () => {
expect(computeGenerationTps(0, 5000, 100)).toBe(0)
})
it('returns 0 when generation time would be zero or negative', () => {
// pathological: totalTime === firstTokenTime → division by zero guarded
expect(computeGenerationTps(50, 1000, 1000)).toBe(0)
// even more pathological: firstTokenTime > totalTime (shouldn't happen,
// but guard anyway)
expect(computeGenerationTps(50, 1000, 1500)).toBe(0)
})
it('returns 0 when totalTime is 0', () => {
// never-started stream
expect(computeGenerationTps(0, 0, 0)).toBe(0)
})
it('handles small TTFT relative to totalTime', () => {
// remote/fast model: 100 tokens, 1000ms total, 50ms TTFT
// generation = 950ms → 100/950*1000 ≈ 105.26 tok/s
const tps = computeGenerationTps(100, 1000, 50)
expect(tps).toBeCloseTo(105.263, 2)
})
it('handles single token (degenerate but valid)', () => {
// 1 token at 500ms TTFT, total 510ms → generation 10ms → 100 tok/s
// Math is correct but practically meaningless; we don't special-case it.
const tps = computeGenerationTps(1, 510, 500)
expect(tps).toBe(100)
})
})
// ── token economy (B6 — 2026-08-05) ──────────────────
describe('getAverageTokens', () => {
it('averages the total token spend, rounded', () => {
const r = {
m: [makeResult('m', 40, { totalTokens: 5000 }), makeResult('m', 40, { totalTokens: 8000 })],
}
expect(getAverageTokens(r, 'm')).toBe(6500)
})
it('is null for a model with no runs', () => {
expect(getAverageTokens({}, 'unknown')).toBeNull()
})
})
describe('getAccuracy', () => {
it('is the fraction correct over runs that recorded a verdict', () => {
const r = {
m: [
makeResult('m', 40, { correct: true }),
makeResult('m', 40, { correct: false }),
makeResult('m', 40, { correct: true }),
makeResult('m', 40, {}), // pre-2.6.3 run, no verdict, excluded
],
}
expect(getAccuracy(r, 'm')).toBeCloseTo(2 / 3, 5)
})
it('is null when no run recorded a verdict', () => {
expect(getAccuracy({ m: [makeResult('m', 40)] }, 'm')).toBeNull()
})
})
describe('getAverageThinkShare', () => {
it('averages thinkTokens over totalTokens', () => {
const r = {
m: [
makeResult('m', 40, { totalTokens: 100, thinkTokens: 40 }), // 0.4
makeResult('m', 40, { totalTokens: 200, thinkTokens: 20 }), // 0.1
],
}
expect(getAverageThinkShare(r, 'm')).toBeCloseTo(0.25, 5)
})
it('ignores runs that recorded no think count', () => {
expect(getAverageThinkShare({ m: [makeResult('m', 40, { totalTokens: 100 })] }, 'm')).toBeNull()
})
})
describe('getTruncatedCount', () => {
it('counts only runs cut off by the token budget', () => {
const r = {
m: [
makeResult('m', 40, { finishReason: 'length' }),
makeResult('m', 40, { finishReason: 'stop' }),
makeResult('m', 40, { finishReason: 'length' }),
makeResult('m', 40, {}),
],
}
expect(getTruncatedCount(r, 'm')).toBe(2)
})
})
describe('getLeaderboard carries the economy stats', () => {
it('reports tokens, accuracy, think share and truncation next to t/s', () => {
const r = {
m: [makeResult('m', 40, { totalTokens: 100, thinkTokens: 30, correct: true, finishReason: 'length' })],
}
const [entry] = getLeaderboard(r)
expect(entry.avgTps).toBe(40)
expect(entry.avgTokens).toBe(100)
expect(entry.accuracy).toBe(1)
expect(entry.thinkShare).toBeCloseTo(0.3, 5)
expect(entry.truncated).toBe(1)
})
it('leaves the new stats null when the runs predate them', () => {
const [entry] = getLeaderboard({ m: [makeResult('m', 40, { totalTokens: 200 })] })
expect(entry.accuracy).toBeNull()
expect(entry.thinkShare).toBeNull()
expect(entry.truncated).toBe(0)
})
})
})
describe('toMarkdownReport economy columns', () => {
it('has the economy header and fills a row', () => {
const r = { fast: [makeResult('fast', 44, { totalTokens: 5000, thinkTokens: 1000, correct: true })] }
const md = toMarkdownReport(r, '2026-08-05 12:00')
expect(md).toContain('Avg tokens')
expect(md).toContain('Think')
expect(md).toContain('Correct')
expect(md).toContain('| 5000 |') // avg tokens cell
expect(md).toContain('20%') // 1000 / 5000 thinking
expect(md).toContain('100%') // accuracy
})
it('marks a truncated run and dashes a stat it never recorded', () => {
const r = { m: [makeResult('m', 40, { totalTokens: 900, finishReason: 'length' })] }
const md = toMarkdownReport(r, '2026-08-05 12:00')
expect(md).toContain('900 (1 cut off)')
expect(md).toMatch(/\| 900 \(1 cut off\) \| - \| - \|/)
})
})