Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Models page: the chosen conversation context and cache format count in the memory estimate (#3832)

Niko1221f750086183f3
diff --git a/apps/host/src/components/NativeModels.tsx b/apps/host/src/components/NativeModels.tsx
index 9523827c1..4bf16382c 100644
--- a/apps/host/src/components/NativeModels.tsx
+++ b/apps/host/src/components/NativeModels.tsx
@@ -204,7 +204,7 @@ export function NativeModels({ onOtherModels, onReady, onImport, onConnectEngine
{tr(expert && IMAGE_GENERATION_VISIBLE ? <><details className="native-generation"><summary>{tr("Image generation ")}<span className="muted">{tr("Optional")}</span></summary><RunImageModel library={imageLibrary} disabled={pending || !!active} onChange={() => { }}/></details></> : null)}
{tr(expert ? <section id="native-expert-settings" ref={expertSection} className="st-card native-execution"><h2 className="h4">{tr("Expert settings")}</h2>{tr(tunedNow ? <p className="small muted native-tuned-note">{tr("Showing the values of your tuning '{0}'.", [tunedNow.name])}</p> : null)}<label className="native-toggle"><span><strong>{tr("Run multiple models")}</strong><small>{tr("Loading another model keeps the others loaded.")}</small></span><button className="st-toggle" role="switch" aria-label={tr("Run multiple models")} aria-checked={multiple} onClick={() => setMultiple(v => !v)}/></label>{tr(cardsGpus.length > 1 ? <label className="input-field"><span>{tr("Layer split")}</span><input className="st-input" value={split} placeholder={tr("auto or 16,32")} onChange={e => setSplit(e.target.value)}/></label> : null)}<div className="native-field-grid"><label className="input-field"><span>{tr("Context memory")}</span><Select ariaLabel={tr("Native KV format")} value={kv} options={[{ value: 'int8', label: 'INT8' }, { value: 'q4_0', label: 'Q4' }, { value: 'k8v4', label: 'K8 / V4' }]} onChange={setKv}/></label><label className="input-field"><span>{tr("Low-RAM mode")}</span><Select ariaLabel={tr("Native low RAM mode")} value={lowRam} options={['auto', 'resident', 'mmap'].map(v => ({ value: v, label: v === 'auto' ? 'Automatic' : v === 'resident' ? 'Keep experts in RAM' : 'Read experts from disk' }))} onChange={setLowRam}/></label></div>{tr(installed ? <><NativeSettingsFile model={installed.model}/><NativeSettings model={installed.model}/></> : <p className="small muted">{tr("More engine parameters become available after preparing this version.")}</p>)}</section> : null)}
</div>
- <aside className="stack stack--lg"><PcFitCard gpuName={pcGpu?.name} ramGb={pcRam} vramGb={pcVram} inUse={keptModels.length ? { ramGb: Math.round(keptRam), vramGb: Math.round(keptVramOn(chosenCards)) } : null} fit={selectedSize ? fitOn(selectedSize, fitHardware) : null}/><section className="st-card"><h2 className="h4">{tr("Model files")}</h2><p className="small muted">{tr("Strata prepares all shards, the draft layer and shared files. Completed downloads are reused.")}</p><code className="native-path">{installed?.dataFolder ?? native?.downloadDirectory ?? native?.installed?.data_folder ?? 'Choose an installation'}</code><Btn size="sm" variant="ghost" onClick={() => { setSettingsSection('Storage'); go('settings'); }}>{tr("Storage settings")}</Btn></section><Btn size="sm" variant="ghost" onClick={onOtherModels}>{tr("Other local models\u2026")}</Btn></aside>
+ <aside className="stack stack--lg"><PcFitCard gpuName={pcGpu?.name} ramGb={pcRam} vramGb={pcVram} inUse={keptModels.length ? { ramGb: Math.round(keptRam), vramGb: Math.round(keptVramOn(chosenCards)) } : null} fit={selectedSize ? fitOn(selectedSize, fitHardware, true, { context, kv }) : null}/><section className="st-card"><h2 className="h4">{tr("Model files")}</h2><p className="small muted">{tr("Strata prepares all shards, the draft layer and shared files. Completed downloads are reused.")}</p><code className="native-path">{installed?.dataFolder ?? native?.downloadDirectory ?? native?.installed?.data_folder ?? 'Choose an installation'}</code><Btn size="sm" variant="ghost" onClick={() => { setSettingsSection('Storage'); go('settings'); }}>{tr("Storage settings")}</Btn></section><Btn size="sm" variant="ghost" onClick={onOtherModels}>{tr("Other local models\u2026")}</Btn></aside>
</div>
<Dialog open={!!plan} title={tr(installed?.ready ? "Apply settings and load" : "Download and prepare")} compact onClose={() => !pending && setPlan(null)} actions={<><Btn disabled={pending} onClick={() => setPlan(null)}>{tr("Cancel")}</Btn><Btn variant="primary" disabled={pending || !!plan?.plan?.disk_short} onClick={() => void download()}>{tr(installed?.ready ? "Apply & load" : "Download & load")}</Btn></>}><p><strong>{tr(plan?.plan?.title ?? selectedFamily?.title)}</strong>{tr(installed?.ready ? null : <>{tr(" \u00B7 approximately ")}{tr(plan?.plan?.download_gb ?? selectedSize?.download_gb)}{tr(" GB")}</>)}</p>{tr(plan?.drafting?.needed ? <p className="small muted">{tr("Choosing DFlash2 also downloads its drafter: about {0} GB, once. If it does not fit in graphics memory, the model uses the MTP instead.", [plan.drafting.gb])}</p> : null)}{tr(installed?.ready ? <p className="small muted">{tr("The model files are already on this computer. Nothing will be downloaded; only the settings you chose are applied.")}</p> : null)}{tr(plan?.plan?.disk_short ? <ErrorNotice message={plan.plan.disk_short}/> : null)}{tr(installed?.ready ? <p className="small muted">{tr("Any currently loaded native models will be unloaded first.")}</p> : <><p className="small muted">{tr("Existing files are checked and reused. Any currently loaded native models will be unloaded before preparing files.")}</p><p className="small muted">{tr("Your computer may feel slower while weights load. You can stop the download and resume later.")}</p></>)}{tr(selectedSize?.images ? <p className="small muted">{tr("Vision encoder files are included. Picture reading loads only when enabled.")}</p> : null)}</Dialog>
diff --git a/apps/host/src/lib/native-fit.ts b/apps/host/src/lib/native-fit.ts
index 23dad9e6d..b34d27849 100644
--- a/apps/host/src/lib/native-fit.ts
+++ b/apps/host/src/lib/native-fit.ts
@@ -30,16 +30,27 @@ export function memoryNeeded(v: FitInput): number {
return round1((v.experts_gb ?? v.ram_needed_gb ?? 0) + DENSE_EXTRA_GB);
}
-export function fitOn(v: FitInput, hw: FitHardware | null | undefined, mtp = true): Fit {
+/** The conversation cache of Flash-Next: 13 attention layers, bytes per token per layer by cache format, 5.4% over the bare cache
+ * (strata_control/vram_fit.py KV_BYTES, KV_MEASURED_FACTOR; setup.py UD_FIT_KV_BYTES). */
+const KV_BYTES: Record<string, number> = { fp16: 2112, int8: 1056, q4_0: 576, k8v4: 816 };
+const KV_LAYERS = 13, KV_FACTOR = 1.054, BASE_CONTEXT = 32768;
+export function cacheGb(context: number, kv = 'int8'): number {
+ return context * KV_LAYERS * (KV_BYTES[String(kv).toLowerCase()] ?? KV_BYTES.int8) * KV_FACTOR / 2 ** 30;
+}
+export interface CacheChoice { context: number; kv?: string }
+
+export function fitOn(v: FitInput, hw: FitHardware | null | undefined, mtp = true, cache?: CacheChoice | null): Fit {
const memoryGb = memoryNeeded(v);
+ // DENSE_GB already holds a 32K cache in the default format; the person's context and format replace it (#3832).
+ const denseGb = cache && cache.context > 0 ? Math.max(DENSE_GB - cacheGb(BASE_CONTEXT), 0) + cacheGb(cache.context, cache.kv) : DENSE_GB;
const ram = Number(hw?.ram_gb), vram = Number(hw?.vram_gb) || 0;
if (!Number.isFinite(ram) || ram <= 0) return { memoryGb, vramUseGb: 0, ramUseGb: 0, spillGb: 0, known: false };
const experts = Math.max(0, memoryNeeded(v) - DENSE_EXTRA_GB);
- const onGpu = Math.max(0, Math.min(experts, vram - DENSE_GB - (mtp && vram > 0 ? MTP_VRAM_GB : 0)));
+ const onGpu = Math.max(0, Math.min(experts, vram - denseGb - (mtp && vram > 0 ? MTP_VRAM_GB : 0)));
const ramRoom = Math.max(0, ram - SYSTEM_ROOM_GB);
const spillGb = Math.max(0, experts - onGpu - ramRoom);
const inRam = Math.max(0, experts - onGpu - spillGb);
- const vramUseGb = vram > 0 ? Math.min(vram, DENSE_GB + (mtp ? MTP_VRAM_GB : 0) + onGpu) : 0;
+ const vramUseGb = vram > 0 ? Math.min(vram, denseGb + (mtp ? MTP_VRAM_GB : 0) + onGpu) : 0;
return { memoryGb, vramUseGb: round1(vramUseGb), ramUseGb: round1(inRam + DENSE_EXTRA_GB * (vram > 0 ? 0 : 1)), spillGb: round1(spillGb), known: true };
}
diff --git a/apps/host/tests/native-fit.test.mjs b/apps/host/tests/native-fit.test.mjs
index b09e2b2d2..b1fb57fd3 100644
--- a/apps/host/tests/native-fit.test.mjs
+++ b/apps/host/tests/native-fit.test.mjs
@@ -5,7 +5,7 @@ import { readFile } from 'node:fs/promises';
import { transform } from 'esbuild';
const { code } = await transform(await readFile(new URL('../src/lib/native-fit.ts', import.meta.url), 'utf8'), { loader: 'ts', format: 'esm' });
-const { fitOn, memoryNeeded, RECOMMENDED, VERSION_GROUPS, groupOpen, recommendedVersions, PICK_NOTE, OTHER_NOTE, MTP_VRAM_GB } = await import(`data:text/javascript;base64,${Buffer.from(code).toString('base64')}`);
+const { fitOn, cacheGb, memoryNeeded, RECOMMENDED, VERSION_GROUPS, groupOpen, recommendedVersions, PICK_NOTE, OTHER_NOTE, MTP_VRAM_GB } = await import(`data:text/javascript;base64,${Buffer.from(code).toString('base64')}`);
const swift = { ram_needed_gb: 48, experts_gb: 35.5 };
const gyroS = { ram_needed_gb: 27.6, experts_gb: 25.8, mem_total: true };
@@ -83,3 +83,15 @@ test('the tiers of the engine are 40 GB Swift IQ2_XS, 48 GB IQ3_XXS, 64 GB IQ3_S
assert.deepEqual([...m[1].matchAll(/\(([\d.]+),\s*"(\w+)",\s*"(\w+)"\)/g)].map(x => [Number(x[1]), x[2], x[3]]), [[62, 'qwen', 'IQ3_S'], [46, 'qwen', 'IQ3_XXS'], [38, 'swift', 'IQ2_XS']], file);
}
});
+
+test('the conversation context and cache format move the estimate (#3832)', () => {
+ const pc = { ram_gb: 64, vram_gb: 16 };
+ const base = fitOn(swift, pc);
+ assert.deepEqual(fitOn(swift, pc, true, { context: 32768, kv: 'int8' }), base, 'the default context is what the fixed figure already held');
+ const big = fitOn(swift, pc, true, { context: 524288, kv: 'int8' });
+ assert.ok(big.vramUseGb >= base.vramUseGb && big.ramUseGb > base.ramUseGb, 'a bigger cache takes graphics memory from the experts, which then sit in RAM');
+ assert.ok(fitOn(swift, pc, true, { context: 524288, kv: 'q4_0' }).ramUseGb < big.ramUseGb, 'a smaller cache format leaves more room');
+ assert.ok(Math.abs(cacheGb(524288, 'int8') * 1024 - 7361) < 25, 'the 512K Int8 cache is the 7361 MiB the engine reports');
+ const tight = fitOn(q4, pc, true, { context: 524288, kv: 'int8' });
+ assert.ok(tight.spillGb > fitOn(q4, pc).spillGb, 'what no longer fits is read from the SSD and the card says so');
+});