Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Models: a per-model draft layer (MTP) switch, picture-off and custom context that stick

Niko12217a2bfc52ae1d
diff --git a/apps/host/server/native-control.mjs b/apps/host/server/native-control.mjs
index 147ff2255..94bf6c571 100644
--- a/apps/host/server/native-control.mjs
+++ b/apps/host/server/native-control.mjs
@@ -6,7 +6,7 @@ import { existsSync, readdirSync, readFileSync, writeFileSync, mkdirSync, realpa
import { join, resolve, dirname, basename } from 'node:path';
import { fileURLToPath } from 'node:url';
import { createServer } from 'node:net';
-import { NATIVE_FIELDS, validateNativeSettings, validateMcpServers, publicMcpServers, publicNativeEnv, restoreNativeEnv } from './native-settings.mjs';
+import { NATIVE_FIELDS, APP_OWNED_FIELDS, withoutDraftLayer, validateNativeSettings, validateMcpServers, publicMcpServers, publicNativeEnv, restoreNativeEnv } from './native-settings.mjs';
import { resourceSettings, saferStart } from './native-resources.mjs';
import { resolvePython, venvPython } from './python-resolver.mjs';
import os from 'node:os';
@@ -53,7 +53,7 @@ export function createNativeControl({directory,requests=()=>[],changed=()=>{},sp
function state(){return {...snapshot,downloadDirectory:preferences.downloadDirectory??(modelsDirectory?modelsDirectory():snapshot.installed?.data_folder),fields:nativeFields(),instances:[...instances.values()].map(({key,configPath,fingerprint,mcpServers,...i})=>structuredClone(i)),defaultId,jobs:[...jobs.values()].slice(-12).map(({process,abort,engineAbort,rawResult,...j})=>j),capabilities:{responses:sourceText().includes('/v1/responses'),updates:!!snapshot.available,rollback:previousEngine(),componentUpdates:packaged&&typeof control.update==='function',calibration:packaged?false:snapshot.hardware?.gpus?.some(g=>/nvidia/i.test(g.name??''))??false}};}
let sourceCache={path:'',time:0,text:''};
function sourceText(){try{const path=join(packaged?control.control.dir:preferences.repo,'serve/server.py'),time=statSync(path).mtimeMs;if(sourceCache.path===path&&sourceCache.time===time)return sourceCache.text;sourceCache={path,time,text:readFileSync(path,'utf8')};return sourceCache.text;}catch{return '';}}
- function nativeFields(){const source=sourceText();return NATIVE_FIELDS.map(field=>({...field,supported:field.id.startsWith('vision_')||field.id.startsWith('mcp_')||field.id==='vram_reserve_mib'||source.includes(field.id)}));}
+ function nativeFields(){const source=sourceText();return NATIVE_FIELDS.map(field=>({...field,supported:APP_OWNED_FIELDS.includes(field.id)||field.id.startsWith('vision_')||field.id.startsWith('mcp_')||field.id==='vram_reserve_mib'||source.includes(field.id)}));}
const modelArena=id=>({arenaGb:(snapshot.catalog?.families??[]).flatMap(f=>f.sizes??[]).find(s=>s.id===id)?.experts_gb});
/* A start that ends with `server_exited` carries the engine's last log line with it; the app turns the code into words and
keeps this text under "Details" (src/lib/plain-errors.ts). */
@@ -69,7 +69,7 @@ export function createNativeControl({directory,requests=()=>[],changed=()=>{},sp
for(const name of names){const path=join(dir,name),config=readJson(path,null);if(!config||typeof config!=='object')continue;
const next={...config,...(config.cwd!==undefined?{cwd:fix(config.cwd)}:{}),...(Array.isArray(config.args)?{args:config.args.map(fix)}:{})};
if(JSON.stringify(next)!==JSON.stringify(config)){try{writeModelConfig(path,next);}catch{/* read-only folder: the load reports it */}}}}
- function installedInfo(info){if(!info)return info;return {...info,models:(info.models??[]).map(entry=>{const cfg=/^strata-[a-z0-9_-]+\.json$/i.test(entry.config??'')?readJson(entry.configPath??join(repo(),entry.config)):{};const at=cfg.args?.indexOf('--native')??-1,nativeFile=at>=0?cfg.args[at+1]:null;const dataFolder=typeof nativeFile==='string'&&basename(dirname(dirname(nativeFile)))==='models'?dirname(dirname(dirname(nativeFile))):info.data_folder;return {...entry,diskGb:entry.diskGb??entry.disk_gb,diskBytes:entry.diskBytes??entry.disk_bytes,sharedBytes:entry.sharedBytes??entry.shared_bytes,removableGb:entry.removableGb??entry.removable_gb,dataFolder,visionPrepared:!!cfg.vision,/* Picture reading is ON unless the person turned it off: no saved choice and not loaded is undefined, not false (owner rule: pictures on by default). */visionEnabled:cfg.ui_vision_enabled??(()=>{const running=[...instances.values()].find(i=>i.model===entry.model&&i.status==='ready');return running?!!running.vision:undefined;})(),visionDevice:cfg.ui_vision_device,resourceProfile:cfg.ui_resource_profile,lowRamMode:cfg.args?.includes('--resident-experts')?'resident':cfg.args?.includes('--mmap-experts')?'mmap':'auto'};})};}
+ function installedInfo(info){if(!info)return info;return {...info,models:(info.models??[]).map(entry=>{const cfg=/^strata-[a-z0-9_-]+\.json$/i.test(entry.config??'')?readJson(entry.configPath??join(repo(),entry.config)):{};const at=cfg.args?.indexOf('--native')??-1,nativeFile=at>=0?cfg.args[at+1]:null;const dataFolder=typeof nativeFile==='string'&&basename(dirname(dirname(nativeFile)))==='models'?dirname(dirname(dirname(nativeFile))):info.data_folder;return {...entry,diskGb:entry.diskGb??entry.disk_gb,diskBytes:entry.diskBytes??entry.disk_bytes,sharedBytes:entry.sharedBytes??entry.shared_bytes,removableGb:entry.removableGb??entry.removable_gb,dataFolder,visionPrepared:!!cfg.vision,/* The picture files are prepared when the run config has a vision section: the Models screen then loads straight away instead of running setup again (it used to read 'off' for every model, which sent every load of a picture-capable model through "Apply settings and load", and setup writes the arguments, the draft layer and the reader back each time). */images:cfg.vision?'yes':'off',draftLayer:cfg.mtp!==false,/* Picture reading is ON unless the person turned it off: no saved choice and not loaded is undefined, not false (owner rule: pictures on by default). */visionEnabled:cfg.ui_vision_enabled??(()=>{const running=[...instances.values()].find(i=>i.model===entry.model&&i.status==='ready');return running?!!running.vision:undefined;})(),visionDevice:cfg.ui_vision_device,resourceProfile:cfg.ui_resource_profile,lowRamMode:cfg.args?.includes('--resident-experts')?'resident':cfg.args?.includes('--mmap-experts')?'mmap':'auto'};})};}
function installRoot(){const found=[preferences.installDir].find(d=>d&&existsSync(d));if(found)return realpathSync(found);mkdirSync(defaultInstall(),{recursive:true});return realpathSync(defaultInstall());}
function repo(){if(packaged)return installRoot();const path=realpathSync(preferences.repo);if(!existsSync(join(path,'setup.py'))||!existsSync(join(path,'tools/strata_mcp.py'))||!existsSync(join(path,'serve/server.py')))throw new Error('Choose the Strata repository folder containing setup.py and tools/strata_mcp.py.');return path;}
function python(){return resolvePython({repo:preferences.repo});}
@@ -196,6 +196,9 @@ return {summary:result.summary??'Deleted',removed:result.plan?.gb??null};}
const cfg={...config,...visionPreferences,...(resources?{...resources,ui_resource_profile:args.resourceProfile}:{}),host:'127.0.0.1',open_browser:false,...validateNativeSettings(args.values??{}),vision};
if(args.gpus!==undefined){const detected=new Set((snapshot.hardware?.gpus??[]).map(g=>g.index));if(!Array.isArray(args.gpus)||!args.gpus.length||new Set(args.gpus).size!==args.gpus.length||args.gpus.some(g=>!Number.isInteger(g)||!detected.has(g)))throw new Error('Choose distinct detected GPUs. CPU-only text inference is not supported by this native adapter.');cfg.gpu=args.gpus.length===1?args.gpus[0]:args.gpus;cfg.layer_split=args.gpus.length>1?validateNativeSettings({layer_split:args.layerSplit??'auto'}).layer_split:'auto';if(cfg.layer_split!=='auto'&&cfg.layer_split.split(',').length!==args.gpus.length-1)throw new Error('Choose one layer split point between each pair of GPUs.');}
if(cfg.vram_reserve_mib!==undefined&&cfg.vram_reserve_mib!==null){cfg.args=[...(cfg.args??[])];const at=cfg.args.indexOf('--vram-reserve-mib');if(at>=0)cfg.args.splice(at,2);cfg.args.push('--vram-reserve-mib',String(cfg.vram_reserve_mib));}if(cfg.vision){if(cfg.vision_tokens!=null)cfg.vision={...cfg.vision,max_tokens:cfg.vision_tokens};if(cfg.vision_min_tokens!=null)cfg.vision={...cfg.vision,min_tokens:cfg.vision_min_tokens};}cfg.mcp={...(cfg.mcp??{}),...(cfg.mcp_timeout_s!=null?{timeout_s:cfg.mcp_timeout_s}:{}),...(cfg.mcp_max_rounds!=null?{max_rounds:cfg.mcp_max_rounds}:{}),...(cfg.mcp_max_result_chars!=null?{max_result_chars:cfg.mcp_max_result_chars}:{})};
+ /* The model's own choices win over what setup wrote into the arguments: "Draft layer (MTP)" off leaves --mtp out of every start, and a picture reading that is off leaves --vision out (setup writes both again whenever it runs; the saved settings are applied here, on every load). */
+ if(cfg.mtp===false&&Array.isArray(cfg.args))cfg.args=withoutDraftLayer(cfg.args);
+ if(!cfg.vision&&Array.isArray(cfg.args))cfg.args=cfg.args.filter(a=>a!=='--vision');
for(const arg of cfg.args??[])if(arg.endsWith('.gguf')&&!existsSync(resolve(repo(),arg)))throw new Error('An engine argument names a missing GGUF. Restore the saved arguments or repair the files.');
/* The app's own choice of request slots (4, fewer where the card has no room) is part of the configuration, so a tune measures the engine a load will start: slots take VRAM from the expert cache. */
const parallelChoice=packaged?chooseParallel({config:cfg,values:args.values??{},hardware:snapshot.hardware?.gpus?.length?snapshot.hardware:{gpus:hardwareProbe()??[]}}):null;
diff --git a/apps/host/server/native-settings.mjs b/apps/host/server/native-settings.mjs
index 7a4ed7d21..eebab7f97 100644
--- a/apps/host/server/native-settings.mjs
+++ b/apps/host/server/native-settings.mjs
@@ -7,6 +7,7 @@ export const NATIVE_FIELDS=[
{id:'aliases',label:'API model aliases',group:'Serving',type:'array',hint:'Comma-separated names clients may use for this model.'},
{id:'idle_unload_s',label:'Unload after idle seconds',group:'Serving',type:'number',min:0,max:86400,hint:'Zero keeps the model loaded.'},
{id:'lazy_load',label:'Load on first request',group:'Serving',type:'boolean'},
+ {id:'mtp',label:'Draft layer (MTP)',group:'Serving',type:'boolean',hint:'The draft layer guesses several words ahead, which makes answers faster, and costs about 1 GB of graphics memory and some RAM. Turn it off if the model stops while it starts, or the card is small: the expert cache gets that memory back and answers are slower. Default is on.'},
{id:'batchMtp',label:'MTP per request',group:'Serving',type:'boolean',hint:'Each concurrent request uses the draft layer. Left alone, Strata turns it on for one graphics card with 24 GB or more and several requests at once, and off otherwise; choose on or off here to decide yourself.'},
{id:'parallel',label:'Concurrent requests',group:'Serving',type:'number',min:1,max:8,hint:'More slots need more memory and can slow each response.'},
{id:'engine_silence_s',label:'Engine stall timeout (seconds)',group:'Recovery',type:'number',min:10,max:3600},
@@ -22,6 +23,14 @@ export const NATIVE_FIELDS=[
{id:'mcp_max_rounds',label:'Maximum tool rounds',group:'Tools',type:'number',min:1,max:100},
{id:'mcp_max_result_chars',label:'Maximum tool result characters',group:'Tools',type:'number',min:100,max:1000000},
];
+/** Settings the app itself applies when it starts a model (they are not read from the engine's source, so the engine's text never says it supports them). */
+export const APP_OWNED_FIELDS=['mtp','batchMtp'];
+/** The engine arguments without the draft layer: --mtp <folder or count>, --spec-min-p <p> (its confidence threshold) and --batch-mtp. --spec stays: prompt lookup still uses its windows. Same rule as strata_control/draft_layer.py. */
+export function withoutDraftLayer(args){
+ const out=[];const list=(args??[]).map(String);
+ for(let i=0;i<list.length;i++){const a=list[i];if(a==='--batch-mtp')continue;if(a==='--mtp'||a==='--spec-min-p'){if(i+1<list.length&&!list[i+1].startsWith('--'))i++;continue;}out.push(a);}
+ return out;
+}
export function validateNativeSettings(values){
if(!values||typeof values!=='object'||Array.isArray(values))throw new Error('Native settings must be an object.');
const result={};
diff --git a/apps/host/src/components/NativeModels.tsx b/apps/host/src/components/NativeModels.tsx
index 058273434..454c90038 100644
--- a/apps/host/src/components/NativeModels.tsx
+++ b/apps/host/src/components/NativeModels.tsx
@@ -97,7 +97,7 @@ export function NativeModels({ onOtherModels, onReady, onImport, onConnectEngine
const shownSize = useRef(selectedSize?.id);
useEffect(() => { if (shownSize.current !== selectedSize?.id) { shownSize.current = selectedSize?.id; setCardsKey(null); } }, [selectedSize?.id]);
const values = useMemo(() => ({ family: selectedSize?.sourceFamily, model: selectedSize?.model, context, vision: selectedSize?.images ? vision : 'no', kv, low_ram: lowRam, resourceProfile, ...(expert && gpus.length ? { gpus: gpus.join(',') } : cardsGpus.length > 1 ? { gpus: cardsGpus.join(',') } : cardsGpus.length === 1 ? { gpu: cardsGpus[0] } : {}), loadAfter: true, multiple: expert && multiple }), [selectedSize?.sourceFamily, selectedSize?.model, context, vision, kv, lowRam, resourceProfile, gpus.join(','), cardsGpus.join(','), expert, multiple]);
- const downloadValues = { ...values, vision: selectedSize?.images ? 'yes' : 'no', loadVision: selectedSize?.images ? vision : 'no' };
+ const downloadValues = { ...values, vision: selectedSize?.images && vision !== 'no' ? 'yes' : 'no', loadVision: selectedSize?.images ? vision : 'no' };
useEffect(() => { if (!native?.catalog || family)
return; const current = native.instances.find(i => i.id === native.defaultId); const rec = native.catalog.recommendation ?? { family: null, model: null }; const loadedFamily = families.find(f => f.sizes.some(s => s.id === current?.model)); const have = new Set((native.installed?.models ?? []).filter(m => m.ready).map(m => m.model)); const installedFamily = families.find(f => f.sizes.some(s => have.has(s.id))); const f = loadedFamily ?? installedFamily ?? families.find(f => f.family === rec.family || f.sizes.some(s => s.sourceFamily === rec.family)) ?? families[0]; if (f) {
setFamily(f.family);
@@ -147,7 +147,7 @@ export function NativeModels({ onOtherModels, onReady, onImport, onConnectEngine
setPending(false);
} }
async function load() { if (!selectedSize)
- return; await perform(async () => { if (installed?.ready && context === installed.context && (!selectedSize?.images || installed.images && installed.images !== 'off') && (!installed.kv || kv === installed.kv || kv === tunedNow?.fields.kv) && (lowRam === (installed.lowRamMode ?? 'auto') || lowRam === tunedNow?.fields.lowRam)) {
+ return; await perform(async () => { if (installed?.ready && context === installed.context && (!selectedSize?.images || vision === 'no' || installed.images && installed.images !== 'off') && (!installed.kv || kv === installed.kv || kv === tunedNow?.fields.kv) && (lowRam === (installed.lowRamMode ?? 'auto') || lowRam === tunedNow?.fields.lowRam)) {
const result = await nativeAction<NativeJob>('operations', { kind: 'load', args: { model: selectedSize.id, resourceProfile, vision: selectedSize?.images ? vision : 'no', multiple: expert && multiple, ...(expert && gpus.length ? { gpus, layerSplit: split } : cardsGpus.length ? { gpus: cardsGpus } : {}), values: {} } });
setJobId(result.id);
}
@@ -175,7 +175,7 @@ export function NativeModels({ onOtherModels, onReady, onImport, onConnectEngine
<div className="native-version"><VersionPicker versions={allSizes} selectedId={selectedSize.id} hardware={fitHardware} downloaded={readyIds} onSelect={id => { const f = families.find(f => f.sizes.some(s => s.id === id)); if (f) { setFamily(f.family); setSize(id); setError(''); } }} onOwnGguf={() => (onImport ?? onOtherModels)()}/></div>
{tr(noFit ? <p className="small muted native-spill-note"><Icon name="disk" size={16}/>{tr("May run slower \u00B7 Model data can be read from SSD.")}</p> : null)}
<div className="native-choice__row">
- <div className="native-choice__col"><span className="native-choice__label">{tr("Conversation context")}</span><Select ariaLabel={tr("Conversation context")} value={String(context)} options={(native?.catalog?.contexts ?? [8192, 32768, 65536, 131072, 262144]).map(n => ({ value: String(n), label: `${n / 1024}K` }))} onChange={n => setContext(Number(n))}/></div>
+ <div className="native-choice__col"><span className="native-choice__label">{tr("Conversation context")}</span><Select ariaLabel={tr("Conversation context")} value={String(context)} options={contextChoices(native?.catalog?.contexts ?? [8192, 32768, 65536, 131072, 262144], context).map(n => ({ value: String(n), label: n % 1024 === 0 ? `${n / 1024}K` : `${n.toLocaleString()} tokens` }))} onChange={n => setContext(Number(n))}/></div>
<div className="native-choice__col"><span className="native-choice__label">{tr("Computer use")}</span><div className="native-resource-choice"><button aria-pressed={resourceProfile === 'work'} data-selected={resourceProfile === 'work'} onClick={() => setResourceProfile('work')}><Icon name="computer" size={20}/><span><strong>{tr("Leave 20% for my work")}</strong><small>{tr("Headroom for your other apps")}</small></span>{tr(resourceProfile === 'work' ? <Icon name="check" size={15}/> : null)}</button><button aria-pressed={resourceProfile === 'performance'} data-selected={resourceProfile === 'performance'} onClick={() => setResourceProfile('performance')}><Icon name="bolt" size={20}/><span><strong>{tr("Full performance")}</strong><small>{tr("Give Strata priority")}</small></span>{tr(resourceProfile === 'performance' ? <Icon name="check" size={15}/> : null)}</button></div></div>
</div>
{tr(cardOptions ? <div className="native-cards"><span className="native-choice__label">{tr("Graphics cards")}</span><div className="native-cards__list" role="radiogroup" aria-label={tr("Graphics cards")}>{cardOptions.options.map(o => { const on = activeCards?.key === o.key; return <button key={o.key} type="button" role="radio" aria-checked={on} data-selected={on} disabled={!!active} className="native-cards__opt" onClick={() => setCardsKey(o.key)}><span className="native-cards__dot" aria-hidden="true"/><span><strong>{gpuNumbers(o.label)}</strong><small data-warn={o.verdict === 'spills' || o.verdict === 'tight' || usedBy(o.cards).length > 0}>{[lessKept(o.words, o.cards), ...o.notes, ...(usedBy(o.cards).length ? [tr('Already used by {0}', [usedBy(o.cards).join(', ')])] : [])].join('. ')}.</small></span></button>; })}{cardOptions.ignored.length ? <p className="small muted native-cards__note">{`Not used: ${cardOptions.ignored.map(g => `${g.name ?? 'a card'} (${g.why_ignored})`).join('; ')}.`}</p> : null}</div></div> : null)}
@@ -209,6 +209,10 @@ export function NativeModels({ onOtherModels, onReady, onImport, onConnectEngine
</Screen>;
}
/** setup's "5/7: downloading …" as a sentence a person can read. */
+/** The listed context sizes, plus the model's own when it is a size set by hand in its run config (the select would show nothing, and a load would silently pick another). */
+export function contextChoices(listed: number[], current: number): number[] {
+ return current > 0 && !listed.includes(current) ? [...listed, current].sort((a, b) => a - b) : listed;
+}
function installStepLabel(step?: string): string {
const n = Number(step?.match(/^(\d+)\//)?.[1]);
if (!n)
diff --git a/apps/host/tests/draft-layer.test.mjs b/apps/host/tests/draft-layer.test.mjs
new file mode 100644
index 000000000..0cfd034dd
--- /dev/null
+++ b/apps/host/tests/draft-layer.test.mjs
@@ -0,0 +1,112 @@
+/**
+ * The model's own choices outlive setup: "Draft layer (MTP)" off, picture reading off, a context size set by hand.
+ * Setup writes --mtp, --spec-min-p and --vision into the arguments every time it runs; the host applies the saved choices on every
+ * load. Nothing here starts an engine; Control is a fake that records what the host saved and asked it to start.
+ */
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { createNativeControl } from '../server/native-control.mjs';
+import { withoutDraftLayer, NATIVE_FIELDS, APP_OWNED_FIELDS, validateNativeSettings } from '../server/native-settings.mjs';
+import { checkRunConfig } from '../server/run-config-check.mjs';
+
+const ARGS = ['--pack', '/p', '--expert-cache', 'auto', '--spec', '4', '--spec-min-p', '0.5', '--mtp', '/m/rt', '--max-context', '40000', '--kv', 'int8', '--vision'];
+
+function fixture(config = {}, { gpuGb = 12 } = {}) {
+ const root = mkdtempSync(join(tmpdir(), 'strata-draft-test-'));
+ const install = join(root, 'install'), data = join(root, 'data');
+ mkdirSync(install, { recursive: true });
+ process.env.STRATA_INSTALL_DIR = install;
+ const path = join(install, 'strata-q2_0.json');
+ writeFileSync(path, JSON.stringify({ exe: 'component:strata-engine', tokenizer: join(install, 'tok'), args: ARGS, model_name: 'Qwen q2_0', port: 8080, ...config }));
+ const saved = [];
+ const control = {
+ control: { dir: join(root, 'control') }, version: '0.2.0',
+ async installedModels() { return { install_dir: install, models: [{ id: 'q2_0', name: 'Qwen q2_0', source_config: path, component_status: 'present', context_tokens: 40000 }] }; },
+ async engine() { return { usable: true, backend: 'cuda', components_dir: 'E:\\c', engine: { version: '0.2.0', platform: 'windows-x86_64', backend: 'cuda', exe: 'E:\\e\\strata.exe', root: 'E:\\e' } }; },
+ async saveProfile(name, file) { saved.push(JSON.parse(readFileSync(file, 'utf8'))); return {}; },
+ async start() { return { ok: true, state: 'ready' }; },
+ async stop() { return { ok: true }; },
+ };
+ const native = createNativeControl({ control, hardwareProbe: () => [{ index: 0, vram_gb: gpuGb }], directory: data, spawnProcess: () => { throw new Error('no process'); }, fetcher: async () => Response.json({ status: 'ok' }), requests: () => [] });
+ return { native, saved, path, cleanup: () => rmSync(root, { recursive: true, force: true }) };
+}
+const finish = async (native, id) => { for (let i = 0; i < 500; i++) { const job = native.state().jobs.find(j => j.id === id); if (job.state !== 'running') return job; await new Promise(r => setTimeout(r, 10)); } throw new Error('job did not finish'); };
+const visionSection = { exe: 'E:\\e\\strata-vision.exe', mmproj: 'E:\\mm.gguf', model: 'E:\\m.gguf', gpu: false, max_tokens: 300 };
+
+test('withoutDraftLayer takes out --mtp, --spec-min-p and --batch-mtp and keeps --spec', () => {
+ assert.deepEqual(withoutDraftLayer(ARGS), ['--pack', '/p', '--expert-cache', 'auto', '--spec', '4', '--max-context', '40000', '--kv', 'int8', '--vision']);
+ assert.deepEqual(withoutDraftLayer(['--mtp', '2', '--spec', '2', '--batch-mtp']), ['--spec', '2'], 'Qwen3.6: a draft count');
+ assert.deepEqual(withoutDraftLayer(['--spec', '4', '--mtp']), ['--spec', '4'], 'a flag with no value at the end');
+ assert.deepEqual(withoutDraftLayer(['--mtp', '--spec', '4']), ['--spec', '4'], 'a flag whose next word is another flag');
+});
+
+test('"Draft layer (MTP)" is a setting of the app, not frozen, and "MTP per request" is not frozen either', async () => {
+ assert.ok(NATIVE_FIELDS.some(f => f.id === 'mtp' && f.type === 'boolean'));
+ assert.deepEqual(APP_OWNED_FIELDS.sort(), ['batchMtp', 'mtp']);
+ const f = fixture();
+ try {
+ await f.native.refresh(true);
+ for (const id of ['mtp', 'batchMtp']) assert.equal(f.native.state().fields.find(x => x.id === id).supported, true, `${id} must not be disabled`);
+ assert.deepEqual(validateNativeSettings({ mtp: false }), { mtp: false });
+ assert.equal(checkRunConfig(JSON.stringify({ exe: 'e', tokenizer: 't', args: [], mtp: false })).issues.length, 0, 'the file check knows the key');
+ } finally { await f.native.close(); f.cleanup(); }
+});
+
+test('saving "Draft layer" off keeps it in the model file, and every load starts without --mtp', async () => {
+ const f = fixture();
+ try {
+ await f.native.refresh(true);
+ f.native.save('q2_0', { values: { mtp: false } });
+ assert.equal(JSON.parse(readFileSync(f.path, 'utf8')).mtp, false);
+ // a setup run rewrites the arguments (here: the same text with --mtp back) and keeps the key; the load applies it
+ const job = await finish(f.native, f.native.operation('load', { model: 'q2_0', vision: 'no' }).id);
+ assert.equal(job.state, 'done', job.error);
+ const args = f.saved[0].args;
+ assert.ok(!args.includes('--mtp') && !args.includes('--spec-min-p') && !args.includes('--batch-mtp'), args.join(' '));
+ assert.equal(args[args.indexOf('--spec') + 1], '4', '--spec stays: prompt lookup still uses its windows');
+ assert.equal(args[args.indexOf('--max-context') + 1], '40000', 'a context set by hand is not touched');
+ // and back to the default: the key goes, the draft layer is in again
+ f.native.save('q2_0', { values: { mtp: null } });
+ assert.equal('mtp' in JSON.parse(readFileSync(f.path, 'utf8')), false);
+ } finally { await f.native.close(); f.cleanup(); }
+});
+
+test('the load with the draft layer off in its own values, and a retry, both keep it off', async () => {
+ const f = fixture();
+ try {
+ await f.native.refresh(true);
+ const job = await finish(f.native, f.native.operation('load', { model: 'q2_0', vision: 'no', values: { mtp: false } }).id);
+ assert.equal(job.state, 'done', job.error);
+ assert.ok(!f.saved[0].args.includes('--mtp'));
+ assert.ok(!f.saved[0].args.includes('--batch-mtp'), 'no draft layer, so no draft layer per request either, even on a big card');
+ } finally { await f.native.close(); f.cleanup(); }
+});
+
+test('picture reading off sticks: no vision section and no --vision on the engine, whatever setup wrote', async () => {
+ const f = fixture({ vision: visionSection, ui_vision_enabled: false, ui_vision_device: 'yes' });
+ try {
+ await f.native.refresh(true);
+ const job = await finish(f.native, f.native.operation('load', { model: 'q2_0' }).id); // the screen sends nothing: the saved choice decides
+ assert.equal(job.state, 'done', job.error);
+ assert.equal(f.saved[0].vision, undefined);
+ assert.ok(!f.saved[0].args.includes('--vision'));
+ assert.equal(JSON.parse(readFileSync(f.path, 'utf8')).ui_vision_enabled, false, 'the choice stays saved');
+ } finally { await f.native.close(); f.cleanup(); }
+});
+
+test('picture reading on keeps --vision; the Models screen sees the prepared pictures instead of "off"', async () => {
+ const f = fixture({ vision: visionSection });
+ try {
+ await f.native.refresh(true);
+ assert.equal(f.native.state().installed.models[0].images, 'yes', 'prepared: the screen loads straight away, no setup run');
+ const job = await finish(f.native, f.native.operation('load', { model: 'q2_0' }).id);
+ assert.equal(job.state, 'done', job.error);
+ assert.ok(f.saved[0].args.includes('--vision'));
+ assert.ok(f.saved[0].vision);
+ } finally { await f.native.close(); f.cleanup(); }
+ const g = fixture();
+ try { await g.native.refresh(true); assert.equal(g.native.state().installed.models[0].images, 'off', 'no vision section: not prepared'); } finally { await g.native.close(); g.cleanup(); }
+});
diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md
index 80bf648e7..ad328d0e0 100644
--- a/docs/TROUBLESHOOTING.md
+++ b/docs/TROUBLESHOOTING.md
@@ -57,6 +57,22 @@ the graphics card, and works out how much of the model fits on your GPU. The mou
PC, close other programs (browsers use a lot of RAM) and try again. If it keeps happening, pick a smaller size (Q2_0
or IQ2_XS).
+**The engine stops while it starts, on a small graphics card or a PC with little RAM. Can I turn the draft layer (MTP) off?**
+Yes. The draft layer is what makes answers faster, and it takes about 1 GB of graphics memory (plus some RAM and a memory
+copy that grows with the context). Without it the expert cache gets that memory back and answers are slower. Turn it off
+for one model: in the app, Models, Expert settings, "Draft layer (MTP)": Off; in the Terminal,
+`strata launch --model q2_0 --mtp off` (older Terminals: `--set mtp=off --set spec-min-p=off`); by hand, add `"mtp": false` to the
+model's `strata-<model>.json`. The choice is kept per model and applied on every start.
+
+**I changed `strata-<model>.json` by hand and the change came back.**
+The app and `START-HERE` own the `args` list and the `vision` section of that file: whenever you prepare a model, change its
+context size or its picture setting, or use "Download / repair", setup writes both again from its own choices. It keeps every
+other key (`sampling`, `mcp_servers`, `mtp`, `ui_vision_enabled`, `parallel`, ...) and the earlier file as
+`strata-<model>.json.bak`. So put your choice where it is kept: `"mtp": false` for the draft layer, the picture tick in the
+Models screen (saved as `ui_vision_enabled`), and the context size in the Models screen. Hand-editing `args` still works for
+a model you only start (the `START-HERE` run file and `strata launch` both read the file as it is); setup says which options it
+dropped and where the earlier file is.
+
**It stopped while downloading or installing.**
Run `START-HERE.bat` (Linux: `./setup.sh`) again. It continues where it stopped.
diff --git a/serve/runconfig.py b/serve/runconfig.py
index 62aea1bb7..741680d9f 100644
--- a/serve/runconfig.py
+++ b/serve/runconfig.py
@@ -30,6 +30,8 @@ EDITABLE = [
"Where a non-default reasoning effort goes: start (the default) or end (keeps the cache when it changes)"),
("aliases", "names", "Other model names the server lists and answers to (comma-separated)"),
("idle_unload_s", "num>=0", "Unload the model after this many seconds without requests (0 or empty: never)"),
+ ("mtp", "bool", "Draft layer (MTP): off starts the engine without it (about 1 GB less graphics memory, slower words per second); "
+ "empty or on: as installed"),
("lazy_load", "bool", "Start without loading the model; the first request loads it (text only)"),
("engine_silence_s", "num>=0", "End a request when the engine says nothing for this long (default 300 s, 0 = wait)"),
("api_monitor", "bool", "Keep the last 100 requests' prompts and answers in memory for /api-monitor"),
diff --git a/serve/server.py b/serve/server.py
index 5a119ec7c..fb496abdf 100644
--- a/serve/server.py
+++ b/serve/server.py
@@ -67,6 +67,7 @@ from serve.mcp import McpCancelled, hub_from_config # noqa: E402
from serve import runconfig # noqa: E402
from strata_control import engine_component # noqa: E402
from strata_control import vision_reader # noqa: E402
+from strata_control import draft_layer # noqa: E402
from serve import ready as ready_file # noqa: E402
from serve import owner_guard # noqa: E402
from serve import launch_identity # noqa: E402
@@ -2871,6 +2872,12 @@ def engine_args(cfg: dict) -> list[str]:
config: "auto" by default, or the first layer of each later GPU's share, e.g. "18" or "16,32"; see
layer_split_value)."""
args = list(cfg["args"])
+ # "mtp": false (the Models settings, `strata launch --mtp off`, or by hand): no draft layer, whatever the args say. A setup run
+ # that writes --mtp back into the args cannot undo it, because this is applied at every start.
+ if draft_layer.is_off(cfg):
+ args, removed = draft_layer.strip(args)
+ if removed:
+ print(f'[strata] "mtp": false - starting without the draft layer ({" ".join(removed)} left out)', flush=True)
# #1322: a config with a "vision" section but without --vision in its args (written by an older setup run, or edited by
# hand) advertised images and then refused every picture ("this engine was started without --vision"). The section
# says images are wanted: start the engine with them, and keep the encoder's VRAM free as setup does for a GPU encoder
diff --git a/serve/test_draft_layer.py b/serve/test_draft_layer.py
new file mode 100644
index 000000000..986972400
--- /dev/null
+++ b/serve/test_draft_layer.py
@@ -0,0 +1,51 @@
+"""The draft layer switch: "mtp": false in a run config starts the engine without --mtp (serve/server.py engine_args), however the
+arguments got there, and strata_control/draft_layer.py is the one rule setup, the launcher and the server share.
+
+ python -m unittest serve.test_draft_layer
+"""
+import contextlib
+import io
+import unittest
+
+from serve import runconfig
+from serve.server import engine_args
+from strata_control import draft_layer
+
+ARGS = ["--pack", "/p", "--spec", "4", "--spec-min-p", "0.5", "--mtp", "/m/rt", "--max-context", "32768", "--batch-mtp", "--kv", "int8"]
+
+
+class DraftLayer(unittest.TestCase):
+ def test_strip_keeps_everything_but_the_draft_layer(self):
+ out, removed = draft_layer.strip(ARGS)
+ self.assertEqual(out, ["--pack", "/p", "--spec", "4", "--max-context", "32768", "--kv", "int8"])
+ self.assertEqual(removed, ["--spec-min-p", "--mtp", "--batch-mtp"])
+ self.assertEqual(draft_layer.strip(["--mtp", "2", "--spec", "2"])[0], ["--spec", "2"]) # Qwen3.6: a draft count
+
+ def test_only_an_explicit_false_turns_it_off(self):
+ for cfg in ({}, {"mtp": True}, {"mtp": None}, {"mtp": 0}, {"mtp": "off"}, None):
+ self.assertFalse(draft_layer.is_off(cfg), cfg)
+ self.assertTrue(draft_layer.is_off({"mtp": False}))
+
+ def test_the_server_applies_it_at_every_start(self):
+ on = engine_args({"args": list(ARGS)})
+ self.assertIn("--mtp", on)
+ out = io.StringIO()
+ with contextlib.redirect_stdout(out):
+ off = engine_args({"args": list(ARGS), "mtp": False})
+ for flag in ("--mtp", "--spec-min-p", "--batch-mtp"):
+ self.assertNotIn(flag, off)
+ self.assertEqual(off[off.index("--spec") + 1], "4")
+ self.assertIn("without the draft layer", out.getvalue())
+
+ def test_the_web_settings_page_lists_it(self):
+ cfg = {"args": []}
+ self.assertTrue(any(k["key"] == "mtp" for k in runconfig.view(cfg, "strata-x.json")["keys"]))
+ new, changed = runconfig.apply(cfg, {"mtp": False})
+ self.assertIs(new["mtp"], False)
+ self.assertEqual(changed, ["mtp"])
+ with self.assertRaises(ValueError):
+ runconfig.apply(cfg, {"mtp": "off"})
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/setup.py b/setup.py
index 6f438b629..53d54f418 100644
--- a/setup.py
+++ b/setup.py
@@ -63,6 +63,7 @@ import zipfile
from pathlib import Path
from strata_control import acquisition
+from strata_control import draft_layer
ROOT = Path(__file__).resolve().parent
# The desktop app runs this installer under Strata Control (its bundled Python, STRATA_CONTROL_INSTALL=1). The source
@@ -5416,6 +5417,9 @@ def carry_over(old: dict, cfg: dict) -> list[str]:
nv["mmproj"] = mm
kept.append("vision mmproj")
kept += carry_profile_args(old, cfg)
+ off = draft_layer.apply(cfg) # "mtp": false was the user's (a key setup does not write): the new args honour it
+ if off:
+ kept.append("mtp off (" + " ".join(off) + " left out of args)")
return kept
@@ -7142,6 +7146,10 @@ def main() -> int:
ap.add_argument("--vram-reserve-mib", type=int, metavar="N",
help="VRAM in MiB the engine leaves free for other programs (a game, another model; the engine's "
"default: 700); the expert cache takes that much less")
+ ap.add_argument("--mtp", choices=["on", "off"],
+ help="the draft layer (MTP): off starts the model without it, which frees about 1 GB of graphics memory "
+ "for the expert cache and takes less RAM, at the price of speed; it is kept in the run config "
+ "(\"mtp\": false) and a later setup run does not undo it. on: back to the default")
ap.add_argument("--parallel", type=int, metavar="N",
help="up to N requests decode together (batch slots, opt-in; default: one at a time, the others "
"wait). Each slot takes VRAM from the expert cache; setup says what it recommends")
@@ -8210,6 +8218,15 @@ def main() -> int:
cfg["draft_vocab"] = draft_vocab
if a.browser is not None: # #609: only when given (else an earlier choice is carried over)
cfg["open_browser"] = a.browser
+ if a.mtp is not None: # written only when given (else an earlier choice is carried over)
+ if a.mtp == "off":
+ cfg["mtp"] = False
+ removed = draft_layer.apply(cfg)
+ ok("the draft layer (MTP) is off for this model" + (f" ({' '.join(removed)} left out)" if removed else "") +
+ ": less graphics memory and RAM, slower words per second. `--mtp on` turns it back on")
+ else:
+ cfg["mtp"] = True # explicit, so carry_over does not bring an earlier false back
+ ok("the draft layer (MTP) is on")
# #465: requests at once - written only when given (else an earlier "parallel" is carried over); a recommendation
streaming = "--kv-resident" in args
if a.parallel is not None:
diff --git a/strata_cli/launch/cmd.py b/strata_cli/launch/cmd.py
index 7275b07e2..fba341a87 100644
--- a/strata_cli/launch/cmd.py
+++ b/strata_cli/launch/cmd.py
@@ -89,6 +89,9 @@ def add_parsers(sp: argparse._SubParsersAction) -> list[argparse.ArgumentParser]
"'on' picks the reader by your graphics memory: the CPU reader under 24 GB, the GPU reader from 24 GB (any make); "
"gpu / cpu choose one; off turns pictures off")
g.add_argument("--parallel", type=_parallel, metavar="N", help="requests answered at the same time, 1 to 8 (default: fit to your graphics card)")
+ g.add_argument("--mtp", type=_choice("a draft layer setting", S.MTP_MODES), metavar="on|off",
+ help="the draft layer that makes answers faster. off starts without it: about 1 GB less graphics memory and less RAM, "
+ "slower answers; use it when the engine stops at start on a small card or a PC with little RAM. Default: on")
g.add_argument("--gpu", metavar="LIST", help="graphics card numbers to use, e.g. 0 or 0,1 (default: the engine's choice)")
g.add_argument("--tune", type=_choice("a tuning mode", S.TUNE_MODES), metavar="quick|thorough|skip", help="find the fastest settings first (default: skip here; the guided screens suggest quick the first time)")
g.add_argument("--tune-allow-small", action="store_true", help="let the tuner also try settings that can change answers very slightly")
@@ -194,6 +197,8 @@ def settings_from_args(a: argparse.Namespace, catalog=None) -> tuple[S.LaunchSet
s.vision = a.vision
if a.parallel:
s.parallel = a.parallel
+ if a.mtp:
+ s.mtp = a.mtp
if a.gpu:
s.gpus = _gpus(a.gpu)
if a.tune:
diff --git a/strata_cli/launch/runner.py b/strata_cli/launch/runner.py
index 142981e85..c7d03a22c 100644
--- a/strata_cli/launch/runner.py
+++ b/strata_cli/launch/runner.py
@@ -230,6 +230,12 @@ def derive_config(base: dict, s: S.LaunchSettings, *, bool_flags: frozenset[str]
if log:
cfg["log"] = log
apply_cards(cfg, gpus)
+ if s.mtp == "off": # no draft layer for this launch (serve/server.py applies "mtp": false at every start)
+ from strata_control import draft_layer
+ cfg["mtp"] = False
+ draft_layer.apply(cfg)
+ elif s.mtp == "on":
+ cfg["mtp"] = True
if s.extra:
a_changes, e_changes = S.as_argv_changes(s.extra, bool_flags)
cfg["args"] = tune_knobs.apply_to_argv([str(x) for x in cfg.get("args") or []], a_changes)
diff --git a/strata_cli/launch/settings.py b/strata_cli/launch/settings.py
index ca4779ab8..2a218648a 100644
--- a/strata_cli/launch/settings.py
+++ b/strata_cli/launch/settings.py
@@ -25,6 +25,7 @@ ACCESS_WORDS = {"local": "This computer only", "lan": "My local network", "cloud
"tailscale": "Tailscale"}
TUNE_MODES = ("quick", "thorough", "skip")
VISION_MODES = ("off", "on", "gpu", "cpu")
+MTP_MODES = ("on", "off")
KV_MODES = ("int8", "q4_0", "k8v4", "fp16")
DEFAULT_PORT = 8080
@@ -85,6 +86,7 @@ class LaunchSettings:
kv: str | None = None # None = int8 above 8K, as setup
vision: str | None = None # None = the default (on when the model can read pictures and the engine has the reader), off | on | gpu | cpu
parallel: int | None = None # None = fit to the card
+ mtp: str | None = None # None = as installed (draft layer on); "off" starts without it (less graphics memory and RAM, slower); "on" overrides an installed "off"
tune: str = "skip" # quick | thorough | skip
tune_allow_small: bool = False # the "answer-changing" tick, off by default, as in the app
access: str = "local"
@@ -263,6 +265,8 @@ def flags_text(s: LaunchSettings, *, preset: str | None = None) -> str:
parts.append(f"--vision {s.vision}")
if s.parallel:
parts.append(f"--parallel {s.parallel}")
+ if s.mtp:
+ parts.append(f"--mtp {s.mtp}")
parts.append(f"--tune {s.tune}")
if s.tune_allow_small:
parts.append("--tune-allow-small")
diff --git a/strata_control/draft_layer.py b/strata_control/draft_layer.py
new file mode 100644
index 000000000..f96866e91
--- /dev/null
+++ b/strata_control/draft_layer.py
@@ -0,0 +1,56 @@
+"""The draft layer (MTP) switch of a model's run config.
+
+``"mtp": false`` in a model's run config (strata-<model>.json) starts the engine without the draft layer. The key is the
+person's: setup keeps it when it writes the file again (setup.carry_over), the app's Models settings and `strata launch --mtp off`
+write it, and every start applies it (serve/server.py engine_args), so a setup run that puts ``--mtp <folder>`` back into
+the engine arguments cannot bring the draft layer back.
+
+What the draft layer costs, in plain words: about 1 GB of graphics memory. Its weights are the files in the model data
+folder's ``mtp/rt`` (0.7 GB of experts and 0.1 GB of dense weights, 786 MiB), the engine sets aside another 218 MiB for its
+draft head on a 12 GB card ("expert cache auto: ... +218 MiB for the draft head"), and its own conversation memory grows with
+the context (with a pinned copy of that in RAM when the context streams). The expert cache gets what is left, so every GB here
+is a GB less of cached experts. It buys speed (about twice as many words per second on a
+card that holds the model) and costs that room in the expert cache, so on a small card or a PC with little RAM the saving can be
+worth more than the speed.
+
+Without it the engine still guesses ahead from the text already in the conversation (``--spec`` with prompt lookup), which is
+why ``--spec`` stays and only the draft layer's own options go: ``--mtp``, ``--spec-min-p`` (the draft layer's confidence
+threshold) and ``--batch-mtp`` (the draft layer per request).
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+#: engine options that exist only because of the draft layer: flag -> takes a value
+MTP_FLAGS = {"--mtp": True, "--spec-min-p": True, "--batch-mtp": False}
+
+
+def is_off(cfg: dict[str, Any] | None) -> bool:
+ """True when the run config says the draft layer is off (only an explicit ``false``; absent or true: on)."""
+ return isinstance(cfg, dict) and cfg.get("mtp") is False
+
+
+def strip(args: list[Any]) -> tuple[list[str], list[str]]:
+ """(the arguments without the draft layer's options, the flags that were removed)."""
+ out: list[str] = []
+ removed: list[str] = []
+ items = [str(a) for a in args]
+ i = 0
+ while i < len(items):
+ a = items[i]
+ if a in MTP_FLAGS:
+ removed.append(a)
+ i += 2 if MTP_FLAGS[a] and i + 1 < len(items) and not items[i + 1].startswith("--") else 1
+ continue
+ out.append(a)
+ i += 1
+ return out, removed
+
+
+def apply(cfg: dict[str, Any]) -> list[str]:
+ """Takes the draft layer's options out of ``cfg["args"]`` when the config has it off; returns the flags removed."""
+ if not is_off(cfg) or not isinstance(cfg.get("args"), list):
+ return []
+ cfg["args"], removed = strip(cfg["args"])
+ return removed
diff --git a/tests/terminal/test_launch.py b/tests/terminal/test_launch.py
index 38441b557..39e545290 100644
--- a/tests/terminal/test_launch.py
+++ b/tests/terminal/test_launch.py
@@ -275,6 +275,18 @@ class RunConfig(unittest.TestCase):
self.assertEqual(cfg["env"], {"STRATA_PIN_BY_NEED": "1"})
self.assertEqual(self.BASE["args"][-1], "auto") # the installed config is never edited
+ def test_mtp_off_leaves_the_draft_layer_out_of_this_launch_only(self):
+ base = {**self.BASE, "args": [*self.BASE["args"], "--spec", "4", "--spec-min-p", "0.5", "--mtp", "/m/rt"]}
+ cfg = derive_config(base, S.LaunchSettings(mtp="off"), bool_flags=frozenset())
+ self.assertIs(cfg["mtp"], False)
+ for flag in ("--mtp", "--spec-min-p", "--batch-mtp"):
+ self.assertNotIn(flag, cfg["args"])
+ self.assertEqual(cfg["args"][cfg["args"].index("--spec") + 1], "4") # prompt lookup keeps its windows
+ self.assertIn("--mtp", base["args"]) # the installed config is never edited
+ on = derive_config({**base, "mtp": False}, S.LaunchSettings(mtp="on"), bool_flags=frozenset())
+ self.assertIs(on["mtp"], True) # an explicit on beats an installed off
+ self.assertEqual(derive_config(base, S.LaunchSettings(), bool_flags=frozenset())["args"], base["args"])
+
def test_local_and_cloudflare_stay_on_loopback_without_a_key(self):
for kind in ("local", "cloudflare"):
cfg = derive_config(self.BASE, S.LaunchSettings(access=kind), bool_flags=frozenset())
@@ -659,6 +671,15 @@ class FlagsCommand(unittest.TestCase):
self.assertIn("launch", out)
self.assertIn("connect", out)
+ def test_mtp_flag_is_listed_planned_and_copyable(self):
+ code, out, err = run_cli(["launch", "--help"])
+ self.assertIn("--mtp", out)
+ p = self.plan("--model", "q2_0", "--context", "32k", "--vision", "off", "--mtp", "off")
+ self.assertEqual(p["settings"]["mtp"], "off")
+ self.assertIn("--mtp off", p["flags"])
+ code, out, err = run_cli(["launch", "--plan", "--mtp", "maybe"])
+ self.assertNotEqual(code, 0)
+
def test_vision_help_says_the_picture_rule(self):
code, out, err = run_cli(["launch", "--help"])
text = " ".join(out.split())
diff --git a/tools/strata_mcp.py b/tools/strata_mcp.py
index 7455fa082..a4fa29b8f 100644
--- a/tools/strata_mcp.py
+++ b/tools/strata_mcp.py
@@ -1520,8 +1520,10 @@ def tool_defs(models: dict, families: dict, contexts: list) -> list:
"from the SSD"},
"model": {"type": "string", "enum": list(models),
"description": "the size (quantization): see strata_models"},
- "context": {"type": "integer", "enum": contexts,
- "description": "context length in tokens (more needs more VRAM)"},
+ "context": {"type": "integer", "minimum": 1024, "maximum": 2097152, "examples": contexts,
+ "description": "context length in tokens (more needs more VRAM). Any whole number works, "
+ "for example one typed into a model's run config by hand; the usual sizes are "
+ f"{', '.join(str(c) for c in contexts)}"},
"vision": {"type": "string", "enum": ["no", "yes", "gpu", "cpu"],
"description": "let the model read images (~0.9 GB more download). Default yes where the model can "
"read them (not on a Mac): yes = the best reader for this card (the CPU "
diff --git a/tools/test_setup_config.py b/tools/test_setup_config.py
index d031891c6..daf0ced2e 100644
--- a/tools/test_setup_config.py
+++ b/tools/test_setup_config.py
@@ -147,6 +147,17 @@ class CarryOver(unittest.TestCase):
same = {"args": ["--expert-profile", "/old/expert-profile.bin"]}
self.assertEqual(setup.carry_over(same, new), [])
+ def test_mtp_off_is_the_users_and_survives_a_setup_run_that_writes_the_draft_layer_back(self):
+ old = {"args": ["--spec", "4", "--spec-min-p", "0.5"], "mtp": False}
+ new = {"args": ["--spec", "4", "--spec-min-p", "0.5", "--mtp", "/m/rt", "--max-context", "32768"]}
+ kept = setup.carry_over(old, new)
+ self.assertIs(new["mtp"], False)
+ self.assertEqual(new["args"], ["--spec", "4", "--max-context", "32768"])
+ self.assertTrue(any(k.startswith("mtp off") for k in kept))
+ on = {"args": ["--mtp", "/m/rt"]}
+ setup.carry_over({"args": [], "mtp": True}, on)
+ self.assertEqual(on["args"], ["--mtp", "/m/rt"]) # on: setup's own arguments stay
+
def test_a_missing_mmproj_is_not_carried(self):
old = {"vision": {"mmproj": "/nowhere/mmproj-Q8_0.gguf", "gpu": True}}
new = {"vision": {"mmproj": "/data/mmproj-BF16.gguf", "gpu": True}}
diff --git a/tools/test_strata_mcp.py b/tools/test_strata_mcp.py
index b091f7225..089a286ad 100644
--- a/tools/test_strata_mcp.py
+++ b/tools/test_strata_mcp.py
@@ -291,7 +291,7 @@ class Validation(FakeRoot):
def test_install_arguments(self):
self.assertRejected("strata_install", {"model": "Q9_X"}, "model must be one of")
self.assertRejected("strata_install", {"family": "llama"}, "family must be one of")
- self.assertRejected("strata_install", {"context": 12345}, "context must be one of")
+ self.assertRejected("strata_install", {"context": 100}, "context must be between")
self.assertRejected("strata_install", {"gpus": "0;rm -rf /"}, "gpus=")
self.assertRejected("strata_install", {"gpus": "0 && calc"}, "gpus=")
self.assertRejected("strata_install", {"gpu": -1}, "between")