Project NOMAD is an offline-first knowledge and education server. Wikipedia, thousands of books, courses, maps, and optional local AI, all running on hardware you own with no internet required.
TypeScript
import { test } from 'node:test'
import assert from 'node:assert/strict'
import {
MIN_CONTEXT,
UNKNOWN_BACKEND_CONTEXT,
computeKvBytesPerToken,
estimateKvBytesPerToken,
parseParameterBillions,
parseUserContextCap,
readContextLength,
readModelfileNumCtx,
resolveContextWindow,
snapToLadder,
} from '../../app/utils/context_window.js'
const GB = 2 ** 30
const KIB = 1024
/** Real /api/show model_info for llama3:8b, trimmed to the keys we read. */
const LLAMA3_8B_INFO = {
'general.architecture': 'llama',
'llama.attention.head_count': 32,
'llama.attention.head_count_kv': 8,
'llama.block_count': 32,
'llama.context_length': 8192,
'llama.embedding_length': 4096,
}
test('reads context length via the architecture prefix', () => {
assert.equal(readContextLength(LLAMA3_8B_INFO), 8192)
})
test('falls back to any *.context_length when architecture is missing', () => {
assert.equal(readContextLength({ 'qwen3.context_length': 32768 }), 32768)
})
test('readContextLength copes with a Map, and with nothing', () => {
assert.equal(readContextLength(new Map(Object.entries(LLAMA3_8B_INFO))), 8192)
assert.equal(readContextLength(undefined), undefined)
assert.equal(readContextLength({}), undefined)
assert.equal(readContextLength({ 'llama.block_count': 32 }), undefined)
})
test('reads num_ctx out of the modelfile parameters block', () => {
assert.equal(readModelfileNumCtx('num_keep 24\nnum_ctx 16384\nstop "<|eot_id|>"'), 16384)
assert.equal(readModelfileNumCtx('num_keep 24\nstop "<|eot_id|>"'), undefined)
assert.equal(readModelfileNumCtx(undefined), undefined)
})
test('computes exact KV bytes per token from GGUF metadata', () => {
// 2 (K+V) * 32 layers * 8 kv-heads * 128 head_dim * 2 bytes = 128 KiB/token,
// which puts an 8k window at exactly 1 GiB.
assert.equal(computeKvBytesPerToken(LLAMA3_8B_INFO), 128 * KIB)
assert.equal(computeKvBytesPerToken(LLAMA3_8B_INFO)! * 8192, 1 * GB)
})
test('uses grouped-query KV head count, not attention head count', () => {
// head_count 32 vs head_count_kv 8: using the wrong one overstates cost 4x and
// would hand every GQA model a quarter of the window it can afford.
const noGqa = { ...LLAMA3_8B_INFO, 'llama.attention.head_count_kv': 32 }
assert.equal(computeKvBytesPerToken(noGqa), 4 * computeKvBytesPerToken(LLAMA3_8B_INFO)!)
})
test('KV computation returns undefined on incomplete metadata', () => {
assert.equal(computeKvBytesPerToken({ 'general.architecture': 'llama' }), undefined)
assert.equal(computeKvBytesPerToken({}), undefined)
assert.equal(computeKvBytesPerToken(undefined), undefined)
})
test('KV estimate falls back by parameter size, monotonically', () => {
assert.ok(estimateKvBytesPerToken(1) < estimateKvBytesPerToken(8))
assert.ok(estimateKvBytesPerToken(8) < estimateKvBytesPerToken(70))
// Unknown size is treated as mid-range rather than free.
assert.equal(estimateKvBytesPerToken(undefined), estimateKvBytesPerToken(8))
})
test('parses parameter size from details, falling back to the tag name', () => {
assert.equal(parseParameterBillions('8.0B'), 8)
assert.equal(parseParameterBillions('1.5B'), 1.5)
assert.equal(parseParameterBillions(undefined, 'qwen2.5:0.5b'), 0.5)
assert.equal(parseParameterBillions(undefined, 'llama3.2:3b'), 3)
// The case the old name-regex got wrong: no size in the name at all. Better to
// say "unknown" than to silently assume 8B.
assert.equal(parseParameterBillions(undefined, 'phi3'), undefined)
})
test('snaps down to a ladder rung', () => {
assert.equal(snapToLadder(9000), 8192)
assert.equal(snapToLadder(8192), 8192)
assert.equal(snapToLadder(100), MIN_CONTEXT)
})
test('the model trained context is a hard ceiling', () => {
const d = resolveContextWindow({
modelMaxCtx: 8192,
kvBytesPerToken: 128 * KIB,
availableBytes: 80 * GB,
})
assert.equal(d.contextWindow, 8192)
assert.equal(d.limitedBy, 'model')
})
test('never exceeds trained context even to reach the floor', () => {
// tinyllama trains to 2048. Requesting 4096 pushes RoPE past anything the
// weights saw; a smaller-than-floor window is the correct answer here.
const d = resolveContextWindow({
modelMaxCtx: 2048,
kvBytesPerToken: 22 * KIB,
availableBytes: 8 * GB,
})
assert.equal(d.contextWindow, 2048)
assert.equal(d.limitedBy, 'model')
})
test('memory limits the window when the model would allow more', () => {
// gemma-class KV cost against a small card.
const d = resolveContextWindow({
modelMaxCtx: 262144,
kvBytesPerToken: 1536 * KIB,
availableBytes: 24 * GB,
modelBytes: 7.6 * GB,
})
assert.ok(d.contextWindow < 262144)
assert.ok(d.contextWindow >= MIN_CONTEXT)
})
test('a user cap only ever lowers the result', () => {
const capped = resolveContextWindow({
modelMaxCtx: 131072,
kvBytesPerToken: 112 * KIB,
availableBytes: 24 * GB,
userCap: 8192,
})
assert.equal(capped.contextWindow, 8192)
assert.equal(capped.limitedBy, 'user')
// A cap above what the model or memory allows must not raise anything.
const notRaised = resolveContextWindow({
modelMaxCtx: 8192,
kvBytesPerToken: 128 * KIB,
availableBytes: 24 * GB,
userCap: 131072,
})
assert.equal(notRaised.contextWindow, 8192)
})
test('weights are subtracted from the memory pool', () => {
const inputs = { modelMaxCtx: 131072, kvBytesPerToken: 128 * KIB, availableBytes: 8 * GB }
const withoutWeights = resolveContextWindow(inputs)
const withWeights = resolveContextWindow({ ...inputs, modelBytes: 5 * GB })
assert.ok(withWeights.contextWindow < withoutWeights.contextWindow)
})
test('falls back to a conservative default when nothing is known', () => {
const d = resolveContextWindow({ kvBytesPerToken: 128 * KIB })
assert.equal(d.contextWindow, UNKNOWN_BACKEND_CONTEXT)
assert.equal(d.limitedBy, 'default')
})
test('the resolver is deterministic — the same inputs give the same window', () => {
// Stability is the whole point: a window that varies between requests makes
// Ollama unload and reload the model, stalling the turn and dropping the KV cache.
const inputs = {
modelMaxCtx: 131072,
kvBytesPerToken: 128 * KIB,
availableBytes: 8 * GB,
modelBytes: 4.7 * GB,
}
const first = resolveContextWindow(inputs)
for (let i = 0; i < 5; i++) {
assert.deepEqual(resolveContextWindow(inputs), first)
}
})
test('parses the user context cap setting', () => {
assert.equal(parseUserContextCap('16384'), 16384)
assert.equal(parseUserContextCap('auto'), undefined)
assert.equal(parseUserContextCap(null), undefined)
assert.equal(parseUserContextCap(''), undefined)
assert.equal(parseUserContextCap('nonsense'), undefined)
// Below the floor is meaningless; treat it as unset rather than crippling chat.
assert.equal(parseUserContextCap('512'), undefined)
})