Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

GPU roles in strata launch (--gpu-roles, M/H/O/R/D Cards screen), setup (--gpu-roles, one roles question) and the host's TypeScript/JS rules; golden and setup tests follow the new defaults

Niko12218cfe44b71dfe
diff --git a/apps/host/server/gpu-roles.mjs b/apps/host/server/gpu-roles.mjs
new file mode 100644
index 000000000..f831663aa
--- /dev/null
+++ b/apps/host/server/gpu-roles.mjs
@@ -0,0 +1,52 @@
+/**
+ * The host's copy of the roles rules it needs to WRITE a model's settings (src/lib/gpu-roles.ts has the screen's, strata_control/gpu_roles.py the
+ * Terminal's and the server's; tests/data/gpu_roles_vectors.json is run against all three).
+ * Roles are { [card number]: 'main' | 'helper' | 'off' }. The engine takes the cards in stage order: Helpers, then the Main card LAST.
+ */
+export const ROLES = ['main', 'helper', 'off'];
+const mem = c => Number(c.vram_gb) || 0;
+const speed = c => (typeof c.speed === 'number' && c.speed > 0 ? c.speed : 0);
+const nums = roles => Object.keys(roles).map(Number).sort((a, b) => a - b);
+const onList = roles => nums(roles).filter(i => roles[i] !== 'off');
+const shortName = c => { let n = String(c.name || `GPU ${c.index + 1}`); for (const j of ['NVIDIA GeForce ', 'NVIDIA ', 'GeForce ', 'AMD Radeon ', 'AMD ', 'Intel(R) ', 'Intel ']) if (n.startsWith(j)) { n = n.slice(j.length); break; } return n.trim(); };
+
+export function parseSaved(value) {
+ if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
+ const out = {};
+ for (const [k, v] of Object.entries(value)) {
+ if (!/^\d+$/.test(k.trim()) || !ROLES.includes(String(v).toLowerCase())) return null;
+ out[Number(k)] = String(v).toLowerCase();
+ }
+ if (!Object.keys(out).length || Object.values(out).filter(r => r === 'main').length > 1) return null;
+ return out;
+}
+
+export function normalize(roles, cards) {
+ const cs = new Map(cards.map(c => [c.index, c]));
+ const out = {};
+ for (const i of nums(roles)) if (cs.has(i)) out[i] = ROLES.includes(roles[i]) ? roles[i] : 'off';
+ const list = onList(out), mains = list.filter(i => out[i] === 'main');
+ if (list.length && !mains.length) {
+ const best = [...list].sort((a, b) => speed(cs.get(b)) - speed(cs.get(a)) || Math.round(mem(cs.get(b))) - Math.round(mem(cs.get(a))) || a - b)[0];
+ out[best] = 'main';
+ }
+ for (const extra of mains.slice(1)) out[extra] = 'helper';
+ if (list.length === 1) out[list[0]] = 'main';
+ return out;
+}
+
+export function order(roles, cards) {
+ const cs = new Map(cards.map(c => [c.index, c]));
+ const r = normalize(roles, cards);
+ const helpers = nums(r).filter(i => r[i] === 'helper').sort((a, b) => speed(cs.get(a)) - speed(cs.get(b)) || a - b);
+ return [...helpers, ...nums(r).filter(i => r[i] === 'main')];
+}
+
+/** "gpu_roles" plus the legacy mirror ("gpu" Main last, "gpu_order": "as_given"); nothing on: "gpu": [] and "gpu_none": true. */
+export function mirror(roles, cards) {
+ const r = normalize(roles, cards), lst = order(r, cards);
+ const out = { gpu_roles: Object.fromEntries(nums(r).map(i => [String(i), r[i]])), gpu: [], gpu_cards: {} };
+ if (lst.length) { out.gpu = lst.length === 1 ? lst[0] : lst; out.gpu_order = 'as_given'; } else out.gpu_none = true;
+ for (const c of cards) if (c.index in r) out.gpu_cards[String(c.index)] = [shortName(c), c.bus ?? ''].filter(Boolean).join('|');
+ return out;
+}
diff --git a/apps/host/src/lib/gpu-roles.ts b/apps/host/src/lib/gpu-roles.ts
new file mode 100644
index 000000000..8ba20e054
--- /dev/null
+++ b/apps/host/src/lib/gpu-roles.ts
@@ -0,0 +1,273 @@
+/**
+ * Main / Helper / Off for every graphics card. The same rules as strata_control/gpu_roles.py (the Terminal, setup and the server); both run
+ * tests/data/gpu_roles_vectors.json.
+ *
+ * - Main: the card that runs the LAST stage of the layer split (the output head, the draft layer and the check of its guesses). Exactly one.
+ * - Helper: holds a share of the model, so more of it fits on graphics cards instead of being read from the drive.
+ * - Off: Strata does not touch it.
+ *
+ * The engine takes the cards in stage order (`gpu: [helper, ..., main]`, `gpu_order: "as_given"`), so Main is the LAST one.
+ */
+export type Role = 'main' | 'helper' | 'off';
+export const ROLES: Role[] = ['main', 'helper', 'off'];
+export type Roles = Record<number, Role>;
+
+export interface RoleCard {
+ index: number;
+ name: string;
+ vram_gb: number;
+ /** SMs x clock (or the AMD equivalent): only the order between cards matters. */
+ speed?: number;
+ usable?: boolean;
+ problem?: string | null;
+ vendor?: string;
+ uma?: boolean;
+ integrated?: boolean;
+ unified_gb?: number;
+ /** PCI bus id when known: used to notice a renumbering. */
+ bus?: string;
+}
+
+export const OFF_AT_OR_BELOW_GB = 8;
+const SAME_SPEED = 1.05;
+const MEMORY_GUARD_SPEED = 0.5;
+const MIN_CARD_GB = 4;
+const SPLIT_CARD_OVERHEAD_GB = 2;
+export const CPU_ONLY_LINE = 'Performance is degraded on CPU only.';
+export const NO_CARD_LINE = 'This engine needs a graphics card to start. Turn one on.';
+
+const mem = (c: RoleCard) => Number(c.vram_gb) || 0;
+const speed = (c: RoleCard) => (typeof c.speed === 'number' && c.speed > 0 ? c.speed : 0);
+const shares = (c: RoleCard) => !!(c.integrated || c.uma || c.unified_gb || c.vendor === 'apple');
+const canRun = (c: RoleCard) => c.usable !== false;
+const rnd = (n: number) => Math.round(n);
+
+export function shortName(c: RoleCard): string {
+ let n = String(c.name || `GPU ${c.index + 1}`);
+ for (const junk of ['NVIDIA GeForce ', 'NVIDIA ', 'GeForce ', 'AMD Radeon ', 'AMD ', 'Intel(R) ', 'Intel ']) if (n.startsWith(junk)) { n = n.slice(junk.length); break; }
+ return n.trim();
+}
+/** The number the screens show: GPU 1 is card 0. */
+export const gpuLabel = (c: RoleCard) => `GPU ${c.index + 1}`;
+
+/** The cards shown as rows: every card the engine's make can name, in card order. */
+export function rows<T extends RoleCard>(cards: T[] | null | undefined, vendor?: string | null): T[] {
+ const list = cards ?? [];
+ const make = vendor || pickVendor(list);
+ const mine = make ? list.filter(c => !c.vendor || c.vendor === make) : list;
+ return [...mine].sort((a, b) => a.index - b.index);
+}
+function pickVendor(cards: RoleCard[]): string | null {
+ const makes = new Set(cards.filter(c => canRun(c) && c.vendor).map(c => c.vendor as string));
+ if (makes.size <= 1) return [...makes][0] ?? null;
+ return makes.has('nvidia') ? 'nvidia' : cards[0]?.vendor ?? null;
+}
+
+function eligible(cs: RoleCard[]): RoleCard[] {
+ const real = cs.filter(c => canRun(c) && !shares(c) && mem(c) >= MIN_CARD_GB);
+ return real.length ? real : cs.filter(canRun);
+}
+
+export type MainWhy = 'only' | 'faster' | 'same' | 'memory' | 'none';
+
+export function pickMain(cs: RoleCard[]): { main: RoleCard | null; why: MainWhy } {
+ const pool = eligible(cs);
+ if (!pool.length) return { main: null, why: 'none' };
+ if (pool.length === 1) return { main: pool[0], why: 'only' };
+ const top = Math.max(...pool.map(speed));
+ const group = top > 0 ? pool.filter(c => speed(c) * SAME_SPEED >= top) : [...pool];
+ const byMem = (a: RoleCard, b: RoleCard) => rnd(mem(b)) - rnd(mem(a)) || a.index - b.index;
+ let main = [...group].sort(byMem)[0];
+ let why: MainWhy = group.length === 1 ? 'faster' : 'same';
+ if (top <= 0 && new Set(pool.map(c => rnd(mem(c)))).size > 1) why = 'memory';
+ const bigger = pool.filter(c => rnd(mem(c)) > rnd(mem(main)) && (top <= 0 || speed(c) >= MEMORY_GUARD_SPEED * top));
+ if (bigger.length) { main = [...bigger].sort(byMem)[0]; why = 'memory'; }
+ return { main, why };
+}
+
+export type OffWhy = 'cannot_run' | 'igpu' | 'small' | 'helper';
+export interface Recommendation { roles: Roles; main: number | null; why: MainWhy; reasons: Record<number, MainWhy | OffWhy> }
+
+/** The best card is Main; every other card starts as Helper, EXCEPT a card with 8 GB or less, built-in graphics beside a real card, and a card the engine cannot run: those start Off. */
+export function recommend(cards: RoleCard[] | null | undefined, vendor?: string | null): Recommendation {
+ const cs = rows(cards, vendor);
+ const { main, why } = pickMain(cs);
+ const roles: Roles = {}, reasons: Recommendation['reasons'] = {};
+ for (const c of cs) {
+ const i = c.index;
+ if (main && c === main) { roles[i] = 'main'; reasons[i] = why; }
+ else if (!canRun(c)) { roles[i] = 'off'; reasons[i] = 'cannot_run'; }
+ else if (shares(c) && main) { roles[i] = 'off'; reasons[i] = 'igpu'; }
+ else if (rnd(mem(c)) <= OFF_AT_OR_BELOW_GB) { roles[i] = 'off'; reasons[i] = 'small'; }
+ else { roles[i] = 'helper'; reasons[i] = 'helper'; }
+ }
+ return { roles, main: main ? main.index : null, why, reasons };
+}
+
+const on = (roles: Roles) => Object.keys(roles).map(Number).filter(i => roles[i] !== 'off');
+const byNumber = (roles: Roles) => Object.keys(roles).map(Number).sort((a, b) => a - b);
+
+/** Exactly one Main among the cards that are on; a lone card that is on is Main. */
+export function normalize(roles: Roles, cards: RoleCard[]): Roles {
+ const cs = new Map(cards.map(c => [c.index, c]));
+ const out: Roles = {};
+ for (const i of byNumber(roles)) if (cs.has(i)) out[i] = ROLES.includes(roles[i]) ? roles[i] : 'off';
+ const onList = on(out), mains = onList.filter(i => out[i] === 'main');
+ if (onList.length && !mains.length) {
+ const best = [...onList].sort((a, b) => speed(cs.get(b)!) - speed(cs.get(a)!) || rnd(mem(cs.get(b)!)) - rnd(mem(cs.get(a)!)) || a - b)[0];
+ out[best] = 'main';
+ }
+ for (const extra of mains.slice(1)) out[extra] = 'helper';
+ if (onList.length === 1) out[onList[0]] = 'main';
+ return out;
+}
+
+export interface RoleChange { roles: Roles; note: string | null }
+
+/** One click: the new roles and a code for the quiet line (old_main_helper:<i>, promoted:<i>, only_main). */
+export function setRole(roles: Roles, cards: RoleCard[], idx: number, role: Role): RoleChange {
+ const cs = new Map(cards.map(c => [c.index, c]));
+ const out: Roles = { ...roles };
+ let note: string | null = null;
+ if (!cs.has(idx) || !ROLES.includes(role)) return { roles: out, note };
+ const ids = byNumber(out);
+ const fastest = (list: number[]) => [...list].sort((a, b) => speed(cs.get(b)!) - speed(cs.get(a)!) || a - b);
+ if (role === 'main') {
+ const old = ids.find(i => i !== idx && out[i] === 'main');
+ out[idx] = 'main';
+ if (old !== undefined) { out[old] = 'helper'; note = `old_main_helper:${old}`; }
+ } else if (role === 'helper') {
+ if (out[idx] === 'main') {
+ const others = fastest(ids.filter(i => i !== idx && out[i] === 'helper'));
+ if (!others.length) return { roles: out, note: 'only_main' };
+ out[idx] = 'helper'; out[others[0]] = 'main'; note = `promoted:${others[0]}`;
+ } else out[idx] = 'helper';
+ } else {
+ const wasMain = out[idx] === 'main';
+ out[idx] = 'off';
+ if (wasMain) {
+ const next = fastest(ids.filter(i => out[i] === 'helper'));
+ if (next.length) { out[next[0]] = 'main'; note = `promoted:${next[0]}`; }
+ }
+ }
+ const onList = on(out);
+ if (onList.length === 1) out[onList[0]] = 'main';
+ else if (onList.length && !onList.some(i => out[i] === 'main')) out[onList[0]] = 'main';
+ return { roles: out, note };
+}
+
+/** The engine's card list: Helpers (slowest first, ties by number), then the Main card LAST; Off cards left out. */
+export function order(roles: Roles, cards: RoleCard[]): number[] {
+ const cs = new Map(cards.map(c => [c.index, c]));
+ const r = normalize(roles, cards);
+ const helpers = byNumber(r).filter(i => r[i] === 'helper').sort((a, b) => speed(cs.get(a)!) - speed(cs.get(b)!) || a - b);
+ return [...helpers, ...byNumber(r).filter(i => r[i] === 'main')];
+}
+
+export interface RoleMirror { gpu_roles: Record<string, Role>; gpu: number | number[]; gpu_order?: 'as_given'; gpu_none?: true; gpu_cards: Record<string, string> }
+/** What a writer stores: "gpu_roles" plus the legacy mirror older engines and the Monitor read. Nothing on: "gpu": [] and "gpu_none": true. */
+export function mirror(roles: Roles, cards: RoleCard[]): RoleMirror {
+ const r = normalize(roles, cards), lst = order(r, cards);
+ const out: RoleMirror = { gpu_roles: Object.fromEntries(byNumber(r).map(i => [String(i), r[i]])), gpu: [], gpu_cards: {} };
+ if (lst.length) { out.gpu = lst.length === 1 ? lst[0] : lst; out.gpu_order = 'as_given'; } else out.gpu_none = true;
+ for (const c of cards) if (c.index in r) out.gpu_cards[String(c.index)] = [shortName(c), c.bus ?? ''].filter(Boolean).join('|');
+ return out;
+}
+
+/** Roles of an old "gpu" setting, for display: the LAST card of the engine's order is Main, earlier ones Helpers, unlisted cards Off. */
+export function fromLegacy(gpu: unknown, cards: RoleCard[], gpuOrder?: string | null): Roles {
+ const cs = new Map(cards.map(c => [c.index, c]));
+ let items: number[] = Array.isArray(gpu) ? gpu.map(Number) : gpu === null || gpu === undefined || gpu === '' ? [] : String(gpu).split(',').filter(x => x.trim() !== '').map(Number);
+ items = items.filter(i => cs.has(i));
+ if (!items.length) return { ...recommend(cards).roles };
+ if (items.length > 1 && gpuOrder !== 'as_given') {
+ const sp = items.map(i => speed(cs.get(i)!));
+ if (sp.every(x => x > 0) && Math.max(...sp) >= SAME_SPEED * Math.min(...sp)) items = [...items].sort((a, b) => speed(cs.get(a)!) - speed(cs.get(b)!));
+ }
+ const roles: Roles = {};
+ for (const c of cards) roles[c.index] = 'off';
+ for (const i of items.slice(0, -1)) roles[i] = 'helper';
+ roles[items[items.length - 1]] = 'main';
+ return roles;
+}
+
+export const sameRoles = (a: Roles, b: Roles) => { const ka = byNumber(a), kb = byNumber(b); return ka.length === kb.length && ka.every((k, n) => k === kb[n] && a[k] === b[k]); };
+
+export interface RoleWarning { code: 'igpu_main' | 'cannot_run' | 'low_memory' | 'cpu_only'; card: number | null; text: string; fix: { card: number; role: Role } | null }
+
+/** One short line each, only what a person could not know. Never blocking. */
+export function warnings(roles: Roles, cards: RoleCard[], opts: { spillGb?: number; engineRunsCpuOnly?: boolean } = {}): RoleWarning[] {
+ const cs = new Map(cards.map(c => [c.index, c]));
+ const out: RoleWarning[] = [];
+ for (const i of byNumber(roles)) {
+ const c = cs.get(i), role = roles[i];
+ if (!c || role === 'off') continue;
+ if (!canRun(c)) out.push({ code: 'cannot_run', card: i, text: `${shortName(c)} cannot run Strata.`, fix: { card: i, role: 'off' } });
+ else if (role === 'main' && shares(c) && cards.some(x => !shares(x) && canRun(x))) {
+ const real = cards.filter(x => !shares(x) && canRun(x)).sort((a, b) => speed(b) - speed(a) || mem(b) - mem(a) || a.index - b.index)[0];
+ out.push({ code: 'igpu_main', card: i, text: `${shortName(c)} is built-in graphics; it is slow as Main.`, fix: { card: real.index, role: 'main' } });
+ }
+ }
+ if (!on(roles).length) out.push({ code: 'cpu_only', card: null, text: opts.engineRunsCpuOnly ? CPU_ONLY_LINE : NO_CARD_LINE, fix: null });
+ else if ((opts.spillGb ?? 0) >= 1) out.push({ code: 'low_memory', card: null, text: `Not enough memory for this setup: about ${Math.round(opts.spillGb!)} GB is read from the drive.`, fix: null });
+ return out;
+}
+
+/** The graphics memory the cards that are on give for the model (an extra card costs its own dense layers); 0 when all are off. */
+export function vramOn(roles: Roles, cards: RoleCard[]): number {
+ const cs = new Map(cards.map(c => [c.index, c]));
+ const onCards = on(roles).map(i => cs.get(i)).filter((c): c is RoleCard => !!c && !shares(c));
+ return Math.max(0, onCards.reduce((n, c) => n + mem(c), 0) - SPLIT_CARD_OVERHEAD_GB * Math.max(0, onCards.length - 1));
+}
+
+export function noteText(code: string | null, cards: RoleCard[]): string {
+ if (!code) return '';
+ const [kind, arg] = code.split(':');
+ const c = arg !== undefined ? cards.find(x => x.index === Number(arg)) : undefined;
+ const name = c ? shortName(c) : 'That card';
+ if (kind === 'old_main_helper') return `${name} is now a Helper.`;
+ if (kind === 'promoted') return `${name} is now Main.`;
+ if (kind === 'only_main') return 'The only card that is on has to be Main.';
+ return '';
+}
+
+/** The one plain line above the cards. */
+export function sentence(cards: RoleCard[] | null | undefined, vendor?: string | null): string {
+ const cs = rows(cards, vendor), rec = recommend(cards, vendor);
+ if (rec.main === null) return 'No graphics card that Strata can use was found.';
+ const byi = new Map(cs.map(c => [c.index, c]));
+ const m = byi.get(rec.main)!;
+ if (cs.length === 1) return `Strata will use your ${shortName(m)}.`;
+ const why = { same: 'the cards are alike, so the lower number is Main', memory: 'it has more memory', faster: 'it is faster', only: 'it is the only card', none: '' }[rec.why];
+ let s = `${shortName(m)} is Main: ${why}.`;
+ const helpers = byNumber(rec.roles).filter(i => rec.roles[i] === 'helper').map(i => shortName(byi.get(i)!));
+ if (helpers.length) s += ` ${helpers.join(' and ')} ${helpers.length > 1 ? 'help' : 'helps'}.`;
+ for (const i of byNumber(rec.roles)) if (rec.roles[i] === 'off') {
+ const code = rec.reasons[i], c = byi.get(i)!;
+ s += ` ${shortName(c)} starts Off: ${code === 'cannot_run' ? 'Strata cannot run it.' : code === 'igpu' ? 'built-in graphics.' : `${OFF_AT_OR_BELOW_GB} GB or less.`}`;
+ }
+ return s;
+}
+
+/** `Main: GPU 2 RTX 4070 Ti. Helper: GPU 1 RTX 3060.` */
+export function describe(roles: Roles, cards: RoleCard[]): string {
+ const cs = new Map(cards.map(c => [c.index, c]));
+ const parts: string[] = [];
+ for (const [role, word] of [['main', 'Main'], ['helper', 'Helper'], ['off', 'Off']] as const) {
+ const names = byNumber(roles).filter(i => roles[i] === role).map(i => (cs.has(i) ? `GPU ${i + 1} ${shortName(cs.get(i)!)}` : `GPU ${i + 1}`));
+ if (names.length) parts.push(`${word}: ${names.join(', ')}.`);
+ }
+ return parts.join(' ');
+}
+
+/** A saved "gpu_roles" object -> roles; null when it is not one. */
+export function parseSaved(value: unknown): Roles | null {
+ if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
+ const out: Roles = {};
+ for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
+ if (!/^\d+$/.test(k.trim()) || !ROLES.includes(String(v).toLowerCase() as Role)) return null;
+ out[Number(k)] = String(v).toLowerCase() as Role;
+ }
+ if (!Object.keys(out).length || Object.values(out).filter(r => r === 'main').length > 1) return null;
+ return out;
+}
diff --git a/apps/host/tests/gpu-roles.test.mjs b/apps/host/tests/gpu-roles.test.mjs
new file mode 100644
index 000000000..1074949ed
--- /dev/null
+++ b/apps/host/tests/gpu-roles.test.mjs
@@ -0,0 +1,74 @@
+/** GPU roles (Main / Helper / Off): src/lib/gpu-roles.ts and server/gpu-roles.mjs run the vectors strata_control/gpu_roles.py runs (tests/data/gpu_roles_vectors.json). */
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { readFile } from 'node:fs/promises';
+import { transform } from 'esbuild';
+
+const { code } = await transform(await readFile(new URL('../src/lib/gpu-roles.ts', import.meta.url), 'utf8'), { loader: 'ts', format: 'esm' });
+const R = await import(`data:text/javascript;base64,${Buffer.from(code).toString('base64')}`);
+const S = await import('../server/gpu-roles.mjs');
+const V = JSON.parse(await readFile(new URL('../../../tests/data/gpu_roles_vectors.json', import.meta.url), 'utf8'));
+const roles = d => d === null || d === undefined ? d : Object.fromEntries(Object.entries(d).map(([k, v]) => [Number(k), v]));
+
+test('recommend: the best card is Main, others Helper, 8 GB or less / built-in / unusable start Off', () => {
+ for (const c of V.recommend) {
+ const r = R.recommend(V.hw[c.hw]);
+ assert.deepEqual(r.roles, roles(c.roles), c.hw);
+ assert.equal(r.main, c.main, c.hw);
+ assert.equal(r.why, c.why, c.hw);
+ }
+});
+
+test('setRole: one click, exactly one Main among the cards that are on', () => {
+ for (const c of V.set_role) {
+ const out = R.setRole(roles(c.roles), V.hw[c.hw], c.card, c.role);
+ assert.deepEqual(out.roles, roles(c.after), JSON.stringify(c));
+ assert.equal(out.note, c.note, JSON.stringify(c));
+ }
+});
+
+test('order: Helpers first, Main LAST, Off left out; one card is a number in the mirror', () => {
+ for (const c of V.order) {
+ assert.deepEqual(R.order(roles(c.roles), V.hw[c.hw]), c.gpu, JSON.stringify(c));
+ assert.deepEqual(S.order(roles(c.roles), V.hw[c.hw]), c.gpu, 'server copy: ' + JSON.stringify(c));
+ }
+ const m = R.mirror({ 0: 'off', 1: 'main' }, V.hw.two);
+ assert.equal(m.gpu, 1);
+ assert.equal(m.gpu_order, 'as_given');
+ assert.deepEqual(R.mirror({ 0: 'helper', 1: 'main' }, V.hw.two).gpu, [0, 1]);
+ const none = R.mirror({ 0: 'off', 1: 'off' }, V.hw.two);
+ assert.deepEqual([none.gpu, none.gpu_none], [[], true]);
+ assert.deepEqual(S.mirror({ 0: 'helper', 1: 'main' }, V.hw.two), R.mirror({ 0: 'helper', 1: 'main' }, V.hw.two));
+ assert.deepEqual(S.mirror({ 0: 'off', 1: 'off' }, V.hw.two), none);
+});
+
+test('warnings: only built-in graphics as Main, a card the engine cannot run, too little memory, CPU only', () => {
+ for (const c of V.warnings) {
+ const w = R.warnings(roles(c.roles), V.hw[c.hw], { spillGb: c.spill_gb ?? 0 });
+ assert.deepEqual(w.map(x => x.code), c.codes, JSON.stringify(c));
+ if (c.fix) assert.deepEqual(w[0].fix, c.fix);
+ for (const x of w) assert.ok(x.text.length < 90, x.text);
+ }
+ // a slower card as Main is understood by people: no warning
+ assert.deepEqual(R.warnings({ 0: 'main', 1: 'helper' }, V.hw.two), []);
+ assert.equal(R.warnings({ 0: 'off', 1: 'off' }, V.hw.two, { engineRunsCpuOnly: true })[0].text, 'Performance is degraded on CPU only.');
+ assert.equal(R.warnings({ 0: 'off', 1: 'off' }, V.hw.two)[0].text, R.NO_CARD_LINE);
+});
+
+test('legacy gpu settings show the roles the engine really uses', () => {
+ for (const c of V.legacy) assert.deepEqual(R.fromLegacy(c.gpu, V.hw[c.hw], c.gpu_order), roles(c.roles), JSON.stringify(c));
+});
+
+test('saved roles: parse; the server copy agrees on which are usable', () => {
+ assert.deepEqual(R.parseSaved({ 0: 'helper', 1: 'main' }), { 0: 'helper', 1: 'main' });
+ for (const bad of [{ 0: 'boss' }, { 0: 'main', 1: 'main' }, 'x', {}, null]) { assert.equal(R.parseSaved(bad), null); assert.equal(S.parseSaved(bad), null); }
+});
+
+test('words: the sentence and the one-line summary', () => {
+ assert.equal(R.sentence(V.hw.one), 'Strata will use your RTX 4070 Ti.');
+ assert.match(R.sentence(V.hw.three), /RTX 4070 Ti is Main: it is faster\./);
+ assert.match(R.sentence(V.hw.three), /RTX 2060 starts Off: 8 GB or less\./);
+ assert.equal(R.describe({ 0: 'helper', 1: 'main' }, V.hw.two), 'Main: GPU 2 RTX 4070 Ti. Helper: GPU 1 RTX 3060.');
+ assert.equal(R.noteText('old_main_helper:1', V.hw.two), 'RTX 4070 Ti is now a Helper.');
+ assert.equal(R.noteText('only_main', V.hw.two), 'The only card that is on has to be Main.');
+});
diff --git a/setup.py b/setup.py
index af4d1e340..c57e93f69 100644
--- a/setup.py
+++ b/setup.py
@@ -1631,34 +1631,58 @@ def choose_gpus(a, found) -> list:
can = together_ok(found)
if not can:
return [single[0]["index"]]
- # the owner's rule (strata_control/gpu_fit.py, shared with the app and `strata launch`): the model goes on the LARGEST card + RAM first;
- # the second card is offered as a layer split and recommended only when the model does not fit one card (with RAM) or would be too tight
- from strata_control import gpu_fit
- mem = None
- d = MODELS.get(getattr(a, "model", None) or "")
- if d:
- mem = float(d["ram_gb"]) if d.get("mem_total") else float(d.get("arena_gb") or d["ram_gb"]) + 1.0
- cards = [{"index": g["index"], "name": g["name"], "vram_gb": g["vram_gb"], "usable": gpu_problem(g) is None} for g in found]
- res = gpu_fit.options(mem if mem is not None else 0.0, ram_gb(), cards)
- by_key = {o["key"]: o for o in res["options"]}
+ return choose_roles(a, found)
+
+
+def card_speeds() -> dict:
+ """{card number: SMs x max clock} of the NVIDIA cards (strata_control/gpu_roles.py); {} when it cannot say."""
+ from strata_control import gpu_roles
+ return gpu_roles.nvidia_speeds()
+
+
+def roles_cards(found) -> list:
+ sp = card_speeds()
+ return [{"index": g["index"], "name": g["name"], "vram_gb": g["vram_gb"], "usable": gpu_problem(g) is None, "vendor": "nvidia",
+ **({"speed": sp[g["index"]]} if g["index"] in sp else {})} for g in found]
+
+
+def choose_roles(a, found) -> list:
+ """Two or more cards can share the model: the recommended role of each (the best card Main, the others Helper, a card with 8 GB or less
+ Off) and ONE question, "Use these roles?". n takes the person's own: 1=main,0=helper (cards left out are Off). Returns the card
+ numbers in the engine's order (Main last) and keeps the roles in a._roles for the config (docs/MULTI_GPU.md)."""
+ from strata_control import gpu_roles
+ cards = roles_cards(found)
+ roles = dict(gpu_roles.recommend(cards)["roles"])
say()
- say(" Strata first tries the model on your largest GPU plus the PC's memory; a second GPU is offered as a layer split")
- say(" (each card holds the experts of its own layers: more room, prompts read about 20% faster; details: docs/MULTI_GPU.md).")
- if mem is None:
- say(" (the model is chosen below; the numbers here are for the recommended size when you name one: --model)")
- opts = [[g for g in found if g["index"] in o["cards"]] for o in res["options"]]
- rec = next((i for i, o in enumerate(res["options"], 1) if o["recommended"]), 1) if mem is not None else 1
- for i, o in enumerate(res["options"], 1):
- say(f" {i}) {o['summary'] if mem is not None else o['label']}" + (" (recommended)" if i == rec else ""))
- for n in o["notes"]:
- say(f" {n}")
+ say(" Strata recommends a role for each graphics card. Main does the last steps of every answer; a Helper holds part of the model;")
+ say(" Off leaves the card alone (a card with 8 GB or less starts Off). Details: docs/MULTI_GPU.md.")
+ by = {c["index"]: c for c in cards}
+ for i in sorted(roles):
+ say(f" GPU {i + 1} {gpu_roles.short_name(by[i]):<22} {by[i]['vram_gb']:>4.0f} GB {roles[i].capitalize()}")
for g in found:
- if gpu_problem(g, together=True) is not None:
- say(f" (GPU {g['index']}, {g['name']}: {gpu_problem(g, together=True)})")
- for g in split_short(can[:2]): # #448: the note for a second card that cannot lend a 4096-token chunk
- say(f" ({split_short_note(g)})")
- pick = opts[int(ask("Which GPUs?", [str(i) for i in range(1, len(opts) + 1)], str(rec), a.yes or a.check)) - 1]
- return [g["index"] for g in pick]
+ if gpu_problem(g, together=True) is not None and roles.get(g["index"]) == "off":
+ say(f" (GPU {g['index'] + 1}, {g['name']}: {gpu_problem(g, together=True)})")
+ if ask("Use these roles?", ["y", "n"], "y", a.yes or a.check) == "n":
+ while True:
+ try:
+ text = input("Roles, e.g. " + ",".join(f"{i}={r}" for i, r in sorted(roles.items())) + " (cards you leave out are Off): ").strip()
+ except EOFError:
+ fail("input ended before a setup answer was received", "run setup in a terminal, or pass --yes to accept the recommended roles")
+ if not text:
+ break
+ try:
+ roles = gpu_roles.roles_from_text(text, cards)
+ break
+ except ValueError as e:
+ say(f" {e}")
+ roles = gpu_roles.normalize(roles, cards)
+ lst = gpu_roles.order(roles, cards)
+ if not lst:
+ fail("every graphics card is Off", "setup needs one card for Main (the app and `strata launch` can start with all cards Off)")
+ a._roles = roles
+ a._roles_cards = cards
+ ok(gpu_roles.describe(roles, cards))
+ return lst
def split_mmap(cfg: dict) -> bool:
@@ -7466,8 +7490,10 @@ def main() -> int:
ap.add_argument("--gpu", help="one GPU, numbered as nvidia-smi numbers them (AMD: as setup lists them) (default: asked when several can be "
"used; with --setup it is saved, when starting it is for that start only)")
ap.add_argument("--gpus", help="several GPUs sharing one model, as nvidia-smi numbers them (AMD: as setup lists "
- "them): \"0,2\", or \"all\" (every card that can); the first is the main one. "
- "Saved, also when starting (see docs/MULTI_GPU.md)")
+ "them): \"0,2\", or \"all\" (every card that can); Strata puts the faster card last, "
+ "and the last card is the Main one. Saved, also when starting (see docs/MULTI_GPU.md)")
+ ap.add_argument("--gpu-roles", metavar="LIST", help="a role for each GPU: main, helper or off, e.g. 1=main,0=helper (cards you do not "
+ "name are off). Main does the last steps of every answer. Saved (docs/MULTI_GPU.md)")
ap.add_argument("--layer-split", help="with --gpus: where each later GPU's layers start (\"18\", \"16,32\"), one "
"rising number per GPU after the first - not layers per card; default "
"auto, placed from each GPU's free VRAM")
@@ -7576,6 +7602,26 @@ def main() -> int:
a.gpu = int(a.gpu)
else:
ap.error(f"--gpu takes a GPU number as nvidia-smi numbers them, e.g. --gpu 1 (or --gpus 0,2), not {a.gpu!r}")
+ a._roles = None
+ if a.gpu_roles:
+ if a.gpu is not None or a.gpus:
+ ap.error("use --gpu-roles (a role for each card) or --gpu / --gpus (which cards), not both")
+ from strata_control import gpu_roles as GR
+ try:
+ named = GR.parse_roles(a.gpu_roles)
+ except ValueError as e:
+ ap.error(f"--gpu-roles: {e}")
+ on = [i for i, r in named.items() if r != "off"]
+ if not on:
+ ap.error("--gpu-roles: setup needs at least one card that is Main or Helper")
+ mains = [i for i, r in named.items() if r == "main"] or [sorted(on)[-1]]
+ helpers = sorted(i for i in on if i != mains[0])
+ a._roles = {**{i: "off" for i in named}, **{i: "helper" for i in helpers}, mains[0]: "main"}
+ order = helpers + [mains[0]]
+ if len(order) > 1:
+ a.gpus = ",".join(str(i) for i in order)
+ else:
+ a.gpu = order[0]
say("Strata - Qwen3.8-Flash-Next on a normal PC (a GPU + system RAM + CPU)")
have = installed_configs()
# A plain start of an installed model finds everything where it already is: it creates, moves and copies nothing
@@ -7769,7 +7815,7 @@ def main() -> int:
gpu_table(found)
sel = choose_gpus(a, found) # asked when two or more cards can share the model
multi = sel if len(sel) > 1 else []
- a.gpu = sel[0] # the main GPU: the checks and the sizing below are its
+ a.gpu = sel[-1] if getattr(a, "_roles", None) else sel[0] # the main GPU: the checks and the sizing below are its (roles: Main is last)
GPU_PICK = a.gpu
gpu = gpu_info(a.gpu)
chosen = [gpu_info(i) for i in sel]
@@ -8592,8 +8638,18 @@ def main() -> int:
if multi: # a layer split across these cards (the server adds the flag)
cfg["gpu"] = multi
cfg["layer_split"] = a.layer_split or "auto"
+ if getattr(a, "_roles", None): # roles chosen: the card list is Main last, in this order, and the server keeps it
+ cfg["gpu_order"] = "as_given"
ok(f"layer split across GPUs {multi} ({cfg['layer_split']})")
recommend_remote_expert_opt(cfg, off=a.no_remote_expert_opt)
+ if getattr(a, "_roles", None): # the roles themselves (strata_control/gpu_roles.py); "gpu" above is their mirror
+ final = cfg["gpu"] if isinstance(cfg.get("gpu"), list) else [cfg["gpu"]] # what the checks above left (a model that cannot share keeps one card)
+ main = final[-1]
+ cfg["gpu_roles"] = {str(i): ("main" if i == main else "helper" if i in final else "off") for i in sorted(set(a._roles) | set(final))}
+ cfg["gpus_asked"] = True
+ cfg.pop("gpu_none", None)
+ if isinstance(cfg.get("gpu"), list) and len(final) > 1:
+ cfg["gpu_order"] = "as_given"
if a.replicas is not None and a.replicas >= 2: # data-parallel replicas (serve/replicas.py), opt-in
cards = multi if multi else []
if not cards or len(cards) % a.replicas:
diff --git a/strata_cli/launch/cmd.py b/strata_cli/launch/cmd.py
index bd401cd3f..4d56fbfb1 100644
--- a/strata_cli/launch/cmd.py
+++ b/strata_cli/launch/cmd.py
@@ -99,6 +99,8 @@ def add_parsers(sp: argparse._SubParsersAction) -> list[argparse.ArgumentParser]
help="the draft layer that makes answers faster. off starts without it: about 1 GB less graphics memory and less RAM, "
"slower answers; use it when the engine stops at start on a small card or a PC with little RAM. Default: on")
g.add_argument("--gpu", metavar="LIST", help="graphics card numbers to use, e.g. 0 or 0,1 (default: the engine's choice)")
+ g.add_argument("--gpu-roles", metavar="LIST", help="a role for each graphics card: main, helper or off, e.g. 1=main,0=helper (cards you do not name are off). "
+ "Main does the last steps of every answer; a helper holds part of the model")
g.add_argument("--tune", type=_choice("a tuning mode", S.TUNE_MODES), metavar="quick|thorough|skip", help="find the fastest settings first (default: skip here; the guided screens suggest quick the first time)")
g.add_argument("--tune-allow-small", action="store_true", help="let the tuner also try settings that can change answers very slightly")
g.add_argument("--set", action="append", default=[], metavar="KEY=VALUE",
@@ -209,8 +211,18 @@ def settings_from_args(a: argparse.Namespace, catalog=None) -> tuple[S.LaunchSet
s.parallel = a.parallel
if a.mtp:
s.mtp = a.mtp
+ if a.gpu and getattr(a, "gpu_roles", None):
+ raise ValueError("Use --gpu (which cards) or --gpu-roles (a role for each card), not both.")
if a.gpu:
s.gpus = _gpus(a.gpu)
+ s.gpu_roles = None
+ if getattr(a, "gpu_roles", None):
+ from strata_control import gpu_roles as GR
+ try:
+ s.gpu_roles = {str(k): v for k, v in GR.parse_roles(a.gpu_roles).items()}
+ except ValueError as e:
+ raise ValueError(f"--gpu-roles: {e}") from None
+ s.gpus = None
if a.tune:
s.tune = a.tune
if a.tune_allow_small:
@@ -304,6 +316,15 @@ def gpu_notices(s: S.LaunchSettings, comp) -> list[S.Notice]:
f"A model cannot be split across {make} and {omake} cards. {gpu_fit.SECOND_MODEL_SENTENCE}",
f"Name a {make} card (" + ", ".join(str(g.get("index")) for g in comp.usable_gpus) + f"), or install the {omake} "
"build of Strata to use that card."))
+ if s.gpu_roles and vendor:
+ from strata_control import gpu_roles
+ have = {int(c.get("index", -1)): c for c in gpu_roles.rows(comp.gpus, vendor)}
+ for k in s.gpu_roles:
+ if int(k) not in have:
+ out.append(S.Notice("error", f"There is no card {k} that this build of Strata can use.",
+ "Cards: " + (", ".join(str(i) for i in sorted(have)) or "none") + "."))
+ if not any(r != "off" for r in s.gpu_roles.values()) and not out:
+ out.append(S.Notice("warn", gpu_roles.NO_CARD_LINE, ""))
note = comp.other_vendor_note
if note:
out.append(S.Notice("warn", note, ""))
diff --git a/strata_cli/launch/runner.py b/strata_cli/launch/runner.py
index 688459800..ee0d1afe1 100644
--- a/strata_cli/launch/runner.py
+++ b/strata_cli/launch/runner.py
@@ -206,11 +206,26 @@ def apply_choices(cfg: dict, s: S.LaunchSettings) -> dict:
return cfg
-def apply_cards(cfg: dict, gpus: list[int] | None) -> None:
+def apply_cards(cfg: dict, gpus: "list[int] | dict | None") -> None:
"""The graphics cards of THIS launch go into its run config, whatever card the model was installed with (no reinstall): one card is a number,
- several are a list with the automatic layer split. None: the installed config's own choice stays."""
+ several are a list with the automatic layer split. None: the installed config's own choice stays. A dict is the roles of every card
+ (strata_control/gpu_roles.mirror: "gpu_roles" plus the legacy "gpu" with the Main card last)."""
+ if isinstance(gpus, dict):
+ for k in ("gpu", "gpu_order", "gpu_none", "gpu_roles", "gpu_cards"):
+ cfg.pop(k, None)
+ cfg.update(gpus)
+ cfg["gpus_asked"] = True
+ cfg.pop("hip_ordinal", None)
+ on = gpus.get("gpu")
+ if isinstance(on, list) and len(on) > 1:
+ cfg["layer_split"] = cfg.get("layer_split") or "auto"
+ else:
+ cfg.pop("layer_split", None)
+ return
if not gpus:
return
+ for k in ("gpu_roles", "gpu_cards", "gpu_none"): # a plain card list replaces an earlier roles choice
+ cfg.pop(k, None)
gpus = [int(g) for g in gpus]
cfg["gpu"] = gpus[0] if len(gpus) == 1 else gpus
cfg["gpus_asked"] = True
@@ -224,7 +239,7 @@ def apply_cards(cfg: dict, gpus: list[int] | None) -> None:
def installed_cards(base: dict | None) -> list[int] | None:
"""The cards the installed model's own config names after a choice (setup --gpu / --gpus, the app's card pick: "gpus_asked"), in its order;
None when it names none or was never asked."""
- if not isinstance(base, dict) or not base.get("gpus_asked"):
+ if not isinstance(base, dict) or not (base.get("gpus_asked") or base.get("gpu_roles")):
return None
g = base.get("gpu")
if g is None or g == "":
@@ -236,27 +251,37 @@ def installed_cards(base: dict | None) -> list[int] | None:
return cards or None
-def cards_for_launch(s: S.LaunchSettings, catalog, ram_gb: float | None = None, base: dict | None = None) -> list[int] | None:
+def cards_for_launch(s: S.LaunchSettings, catalog, ram_gb: float | None = None, base: dict | None = None) -> "list[int] | dict | None":
"""The cards this launch uses: the ones named (--gpu / --gpus, the Cards step), else the cards the installed config was set to (a person's
earlier choice, "gpu": 1 or [1, 0] - never replaced by the automatic pick, which would put the first of equal cards back), else the
recommended ones by the shared fit rule (strata_control/gpu_fit.py) when the computer has more than one card that counts, else None."""
+ if s.gpu_roles:
+ try:
+ from strata_control import gpu_roles
+ comp = catalog.computer()
+ cards = gpu_roles.rows(comp.gpus, comp.engine_vendor)
+ named = {int(k): v for k, v in s.gpu_roles.items()}
+ roles = gpu_roles.normalize({int(c.get("index", 0)): named.get(int(c.get("index", 0)), "off") for c in cards}, cards)
+ return gpu_roles.mirror(roles, cards)
+ except Exception: # noqa: BLE001 - never block a launch over the choice of cards
+ return None
if s.gpus:
return [int(g) for g in s.gpus]
- if installed_cards(base):
+ if installed_cards(base) or (isinstance(base, dict) and base.get("gpu_roles")):
return None
try:
- from strata_control import gpu_fit
+ from strata_control import gpu_roles
comp = catalog.computer()
if len(comp.usable_gpus) < 2:
return None
- row = next((r for r in catalog.rows() if (r.family, r.model) == (s.family, s.model)), None)
- mem = float(row.memory_gb) if row is not None and row.memory_gb else 44.0
- return gpu_fit.recommended_cards(mem, comp.ram_gb, comp.gpus, comp.engine_vendor) or None
+ cards = gpu_roles.rows(comp.gpus, comp.engine_vendor)
+ rec = gpu_roles.recommend(cards, comp.engine_vendor)["roles"]
+ return gpu_roles.order(rec, cards) or None
except Exception: # noqa: BLE001 - never block a launch over the choice of cards
return None
-def derive_config(base: dict, s: S.LaunchSettings, *, bool_flags: frozenset[str], log: str | None = None, gpus: list[int] | None = None) -> dict:
+def derive_config(base: dict, s: S.LaunchSettings, *, bool_flags: frozenset[str], log: str | None = None, gpus: "list[int] | dict | None" = None) -> dict:
"""The private run config of this launch: the installed one plus the choices that are not setup's (host, port, key, cards, advanced)."""
from strata_control import tune_knobs
cfg = json.loads(json.dumps(base))
@@ -578,7 +603,9 @@ class Runner:
if s.model_path:
p = Path(s.model_path)
args += ["--gguf-dir", str(p if p.is_dir() else p.parent)]
- if s.gpus:
+ if s.gpu_roles:
+ args += ["--gpu-roles", ",".join(f"{k}={v}" for k, v in sorted(s.gpu_roles.items(), key=lambda kv: int(kv[0])))]
+ elif s.gpus:
args += ["--gpus", ",".join(str(g) for g in s.gpus)] if len(s.gpus) > 1 else ["--gpu", str(s.gpus[0])]
if sys.platform == "darwin":
# a Mac's setup (Control mode, strata_control/mac_fit.py) takes fewer options (no --kv, --gpu, speed projection)
diff --git a/strata_cli/launch/screens.py b/strata_cli/launch/screens.py
index 917df146e..cae1d7fa8 100644
--- a/strata_cli/launch/screens.py
+++ b/strata_cli/launch/screens.py
@@ -529,13 +529,21 @@ class ModelScreen(Base):
# ------------------------------------------------------------------------------------------------ 2 cards (only with 2+ cards)
class CardsScreen(Base):
+ """Main / Helper / Off for every card (strata_control/gpu_roles.py). The recommendation is already selected; M H O set the role of the
+ card under the cursor, R goes back to the recommendation, D shows what the roles mean."""
+
step_id = "cards"
- help_text = ("Strata tries the model on your largest graphics card plus the computer's memory first. A second card is offered as a layer "
- "split: it adds room and reads prompts faster. The recommended choice is already selected; pick another if you prefer.")
- hints = "↑↓ choose Enter next Esc back ? help"
+ help_text = ("Main does the last steps of every answer; Helper holds part of the model; Off leaves the card alone. "
+ "Keys: M Main, H Helper, O Off, R back to recommended, D details.")
+ hints = "↑↓ choose card M Main H Helper O Off R recommended D details Enter next Esc back ? help"
+ BINDINGS = Base.BINDINGS + [Binding("m", "role('main')", show=False), Binding("h", "role('helper')", show=False),
+ Binding("o", "role('off')", show=False), Binding("r", "recommended", show=False),
+ Binding("d", "details", show=False), Binding("right", "cycle(1)", show=False),
+ Binding("left", "cycle(-1)", show=False)]
def body(self) -> ComposeResult:
yield Static(Text("Which graphics cards should it use?", style=f"bold {ACCENT}"), markup=False)
+ yield Static("", id="reco", markup=False)
yield Menu([("", 0)], id="menu")
yield Static("", id="cardnotes", markup=False)
@@ -552,53 +560,138 @@ class CardsScreen(Base):
except Exception: # noqa: BLE001
return None
- def choices(self) -> dict:
- from strata_control import gpu_fit
- return gpu_fit.options(self.model_memory(), self.ctx.computer.ram_gb, self.ctx.computer.gpus, self.ctx.computer.engine_vendor)
+ def cards(self) -> list[dict]:
+ from strata_control import gpu_roles
+ c = self.ctx.computer
+ return gpu_roles.rows(c.gpus, c.engine_vendor)
+
+ def start_roles(self) -> dict[int, str]:
+ """The roles the screen opens with: this launch's own, else the installed model's own choice, else the recommendation."""
+ from strata_control import gpu_roles
+ s, cards = self.ctx.settings, self.cards()
+ if s.gpu_roles:
+ return gpu_roles.normalize({int(k): v for k, v in s.gpu_roles.items()}, cards)
+ if s.gpus:
+ return gpu_roles.from_legacy(list(s.gpus), cards, "as_given")
+ return self.baseline()
+
+ def baseline(self) -> dict[int, str]:
+ """What "no choice made here" means: the installed model's own cards, else the recommendation."""
+ from strata_control import gpu_roles
+ inst = self.installed()
+ cards = self.cards()
+ if inst:
+ return gpu_roles.from_legacy(inst, cards, "as_given")
+ return dict(gpu_roles.recommend(cards, self.ctx.computer.engine_vendor)["roles"])
def on_mount(self) -> None:
- self.res = self.choices()
+ from strata_control import gpu_roles
+ self.detail = False
+ self.note = ""
+ self.cardlist = self.cards()
+ self.roles = self.start_roles()
+ self.rec = gpu_roles.recommend(self.cardlist, self.ctx.computer.engine_vendor)
+ wide = max((len(gpu_roles.short_name(c)) for c in self.cardlist), default=20)
+ self.query_one(Menu).columns = [("", 8), ("", min(wide + 2, 34)), ("", 10), ("", 32), ("", 0)]
+ self.fill(0)
+ self.paint()
+
+ def role_cell(self, role: str) -> str:
+ words = [("main", "Main"), ("helper", "Helper"), ("off", "Off")]
+ return "".join(f"[ {w} ] " if r == role else f" {w} " for r, w in words)
+
+ def fill(self, index: int = 0) -> None:
+ from strata_control import gpu_roles
items = []
- sel = 0
- cur = self.ctx.settings.gpus
- inst = self.installed()
- for i, o in enumerate(self.res["options"]):
- label = o["label"] + (" (recommended)" if o["recommended"] else "") + (" (as installed)" if inst and sorted(o["cards"]) == sorted(inst) else "")
- items.append(Item([label, Cell(o["words"], WARN if o["verdict"] in ("spills", "tight") else "")], key=o["cards"]))
- if cur is None and (inst is None and o["recommended"] or inst is not None and sorted(o["cards"]) == sorted(inst)) or (cur is not None and o["cards"] == list(cur)):
- sel = i
- wide = max((len(str(i.cells[0])) for i in items), default=40)
- self.query_one(Menu).columns = [("", max(40, min(wide + 3, 70))), ("", 0)]
- self.query_one(Menu).set_items(items, sel)
+ for c in self.cardlist:
+ i = int(c.get("index", 0))
+ role = self.roles.get(i, "off")
+ rec = self.rec["roles"].get(i)
+ tag = "recommended" if rec == role else f"recommended: {rec}"
+ mem = "built-in" if gpu_roles._shares(c) else f"{float(c.get('vram_gb') or 0):.0f} GB"
+ items.append(Item([gpu_roles.gpu_label(c), gpu_roles.short_name(c), mem, self.role_cell(role), Cell(tag, "" if rec == role else MUTE)], key=i))
+ self.query_one(Menu).set_items(items, index)
+
+ def current_card(self) -> int | None:
+ cur = self.menu().current if self.menu() else None
+ return None if cur is None else int(cur.key)
+
+ def action_role(self, role: str) -> None:
+ from strata_control import gpu_roles
+ i = self.current_card()
+ if i is None:
+ return
+ self.roles, code = gpu_roles.set_role(self.roles, self.cardlist, i, role)
+ self.note = gpu_roles.note_text(code, self.cardlist)
+ self.fill(self.menu().index)
+ self.paint()
+
+ def action_cycle(self, step: int) -> None:
+ order = ["main", "helper", "off"]
+ i = self.current_card()
+ if i is None:
+ return
+ self.action_role(order[(order.index(self.roles.get(i, "off")) + step) % 3])
+
+ def action_recommended(self) -> None:
+ self.roles = dict(self.rec["roles"])
+ self.note = ""
+ self.fill(self.menu().index)
+ self.paint()
+
+ def action_details(self) -> None:
+ self.detail = not self.detail
self.paint()
+ def is_cpu_engine(self) -> bool:
+ try:
+ return (self.ctx.catalog.installed_backend() or "") in ("cpu", "metal")
+ except Exception: # noqa: BLE001
+ return False
+
def refresh_body(self) -> None:
+ if not hasattr(self, "roles"):
+ return
+ from strata_control import gpu_fit, gpu_roles
+ c = self.ctx.computer
+ try:
+ self.query_one("#reco", Static).update(Text(gpu_roles.sentence(self.cardlist, c.engine_vendor), style=MUTE))
+ except Exception: # noqa: BLE001
+ return
lines = Text()
- cur = self.menu().current if self.menu() else None
- opt = next((o for o in self.res["options"] if cur is not None and o["cards"] == cur.key), None)
- for n in (opt or {}).get("notes", []):
- lines.append(f" {n}\n", style=MUTE)
- for g in self.res["ignored"]:
- lines.append(f" Not used: {g.get('name')} ({g['why_ignored']}).\n", style=MUTE)
+ if self.note:
+ lines.append(f" {self.note}\n", style=MUTE)
+ spill = gpu_roles.spill_for(self.roles, self.cardlist, self.model_memory(), c.ram_gb)
+ for w in gpu_roles.warnings(self.roles, self.cardlist, spill_gb=spill, engine_runs_cpu_only=self.is_cpu_engine()):
+ fix = ""
+ if w["fix"]:
+ n = gpu_roles.gpu_label(next(x for x in self.cardlist if int(x.get("index", 0)) == w["fix"]["card"]))
+ fix = f" (select {n} and press {w['fix']['role'][0].upper()})"
+ lines.append(f" ! {w['text']}{fix}\n", style=WARN)
+ if self.detail:
+ for word, text in (("Main", "does the last steps of every answer and keeps the conversation. Only one."),
+ ("Helper", "holds a share of the model, so more of it fits on graphics cards."),
+ ("Off", "Strata does not touch the card.")):
+ lines.append(f" {word}: {text}\n", style=MUTE)
+ for g in gpu_fit.for_vendor(c.gpus, c.engine_vendor)[1]:
+ lines.append(f" Not used: {g.get('name')} (another make of card than this engine drives).\n", style=MUTE)
try:
self.query_one("#cardnotes", Static).update(lines)
except Exception: # noqa: BLE001
pass
- def on_cursor(self) -> None:
- self.refresh_body()
-
def explain(self) -> Text:
- return Text("One card plus your RAM is the normal way and keeps things simple. Two cards share the layers: more room for the model and "
- "prompts read about 20% faster, but a small second card can slow it down. Strata recommends one of them; the choice is yours.",
- style=MUTE)
+ return Text("Strata recommends the roles shown. A Helper holds part of the model; Off leaves a card alone (for a game or the screen). "
+ "Any card is one key away; the choice is yours.", style=MUTE)
def commit(self) -> None:
- cur = self.menu().current
- rec = next((o["cards"] for o in self.res["options"] if o["recommended"]), None)
- inst = self.installed()
- keep = rec if inst is None else inst # "let Strata choose" = the recommendation, or the cards the installed model was set to
- self.ctx.settings.gpus = None if keep is not None and sorted(cur.key) == sorted(keep) else list(cur.key) # the recommended choice stays "let Strata choose" (the same rule in setup)
+ if not hasattr(self, "roles"):
+ return
+ s = self.ctx.settings
+ if self.roles == self.baseline():
+ s.gpu_roles, s.gpus = None, None # the recommendation (or the installed choice) stays "not set": a card added later is re-recommended
+ else:
+ s.gpu_roles, s.gpus = {str(i): r for i, r in sorted(self.roles.items())}, None
# ------------------------------------------------------------------------------------------------ 3 folder
diff --git a/strata_cli/launch/settings.py b/strata_cli/launch/settings.py
index 265888796..3b5684656 100644
--- a/strata_cli/launch/settings.py
+++ b/strata_cli/launch/settings.py
@@ -82,6 +82,7 @@ class LaunchSettings:
model_path: str | None = None # "Use model files I already have": a .gguf file or a folder
data_dir: str | None = None # where model files are stored (None: the Strata default)
gpus: list[int] | None = None # None = the engine's choice (all usable cards)
+ gpu_roles: dict[str, str] | None = None # {"1": "main", "0": "helper", "2": "off"}: a role for every card; None = the recommendation
context: int | None = None
kv: str | None = None # None = int8 above 8K, as setup
vision: str | None = None # None = the default (on when the model can read pictures and the engine has the reader), off | on | gpu | cpu
@@ -289,7 +290,9 @@ def flags_text(s: LaunchSettings, *, preset: str | None = None) -> str:
parts.append("--tune-allow-small")
parts.append(f"--access {s.access}")
parts.append(f"--port {s.port}")
- if s.gpus:
+ if s.gpu_roles:
+ parts.append("--gpu-roles " + ",".join(f"{k}={v}" for k, v in sorted(s.gpu_roles.items(), key=lambda kv: int(kv[0]))))
+ elif s.gpus:
parts.append("--gpu " + ",".join(str(g) for g in s.gpus))
for k, v in sorted(s.extra.items()):
parts.append(f"--set {k[2:] if k.startswith('--') else k}={v}" if v else f"--set {k[2:] if k.startswith('--') else k}")
diff --git a/tests/terminal/test_launch.py b/tests/terminal/test_launch.py
index e471c9612..a43b11d07 100644
--- a/tests/terminal/test_launch.py
+++ b/tests/terminal/test_launch.py
@@ -585,7 +585,7 @@ class AdoptRules(unittest.TestCase):
# which cards: named ones win (one card too: --gpu 1 on a box that has them), else the fit rule on a box with several cards
two = catalog(HW_2X16)
self.assertEqual(cards_for_launch(S.LaunchSettings(family="qwen", model="IQ3_XXS", gpus=[1, 0]), two), [1, 0])
- self.assertEqual(cards_for_launch(S.LaunchSettings(family="qwen", model="IQ3_XXS"), two), [0]) # fits one card + RAM
+ self.assertEqual(cards_for_launch(S.LaunchSettings(family="qwen", model="IQ3_XXS"), two), [1, 0]) # roles: card 0 Main (last), card 1 Helper
one = catalog(HW_12)
self.assertEqual(cards_for_launch(S.LaunchSettings(family="qwen", model="IQ3_XXS", gpus=[1]), one), [1])
self.assertIsNone(cards_for_launch(S.LaunchSettings(family="qwen", model="IQ3_XXS"), one))
@@ -614,7 +614,7 @@ class AdoptRules(unittest.TestCase):
self.assertIsNone(installed_cards({**chosen, "gpus_asked": False})) # never asked: setup's default, not a choice
self.assertIsNone(installed_cards({"gpu": 1}))
self.assertIsNone(cards_for_launch(plain, two, base=chosen)) # the installed choice stays
- self.assertEqual(cards_for_launch(plain, two, base={**chosen, "gpus_asked": False}), [0]) # no choice recorded: the fit rule
+ self.assertEqual(cards_for_launch(plain, two, base={**chosen, "gpus_asked": False}), [1, 0]) # no choice recorded: the recommended roles
self.assertEqual(cards_for_launch(S.LaunchSettings(family="qwen", model="IQ3_XXS", gpus=[0]), two, base=chosen), [0]) # --gpu 0 wins
if True:
installed = h.base / "strata-iq3_xxs.json"
diff --git a/tests/terminal/test_launch_tui.py b/tests/terminal/test_launch_tui.py
index e9b0a6f53..f579870bb 100644
--- a/tests/terminal/test_launch_tui.py
+++ b/tests/terminal/test_launch_tui.py
@@ -859,38 +859,58 @@ class SaveAndRun(Base):
class CardsStep(Base):
- def test_cards_step_defaults_to_the_largest_card_plus_ram_and_offers_the_split(self):
+ def test_cards_step_shows_a_role_for_every_card_and_the_recommendation_is_selected(self):
async def case(app, pilot, ctx):
await press(pilot, "enter", "enter")
t = text_of(app)
self.assertIn("Which graphics cards should it use?", t)
self.assertIn("Step 2 of 10", t)
flat = " ".join(t.split())
- self.assertIn("+ RAM", flat) # every choice says what it uses
- self.assertIn("(layer split)", flat) # the second card is offered as a split
+ self.assertIn("[ Main ]", flat) # equal cards: the lower number is Main
+ self.assertIn("[ Helper ]", flat)
self.assertIn("recommended", flat)
- self.assertIn("fits: needs", flat) # plain numbers
- cur = app.screen.menu().current
- self.assertEqual(len(cur.key), 1) # the recommended choice is the single largest card
- await press(pilot, "enter") # keeping it stores "let Strata choose": the same rule runs in setup
+ self.assertIn("M Main", flat)
+ self.assertNotIn("slower", flat) # no explaining warning for a recommendation
+ await press(pilot, "enter") # keeping the recommendation stores "not set": a card added later is re-recommended
+ self.assertIsNone(ctx.settings.gpu_roles)
self.assertIsNone(ctx.settings.gpus)
self.assertIn("Where should the model files be stored?", text_of(app))
self.run_app(case, ctx=make_ctx(HW_2X16))
- def test_choosing_the_split_stores_both_cards(self):
+ def test_m_h_o_set_the_role_of_the_card_under_the_cursor_and_r_goes_back(self):
async def case(app, pilot, ctx):
- await press(pilot, "enter", "enter", "down", "enter")
- self.assertEqual(ctx.settings.gpus, [0, 1])
+ await press(pilot, "enter", "enter")
+ await press(pilot, "down", "o") # card 2 Off
+ flat = " ".join(text_of(app).split())
+ self.assertIn("recommended: helper", flat) # the muted tag says what was recommended
+ await press(pilot, "m") # Main on card 2: card 1 becomes a Helper, one quiet line says so
+ flat = " ".join(text_of(app).split())
+ self.assertIn("is now a Helper.", flat)
+ await press(pilot, "enter")
+ self.assertEqual(ctx.settings.gpu_roles, {"0": "helper", "1": "main"})
+ self.assertIsNone(ctx.settings.gpus)
self.run_app(case, ctx=make_ctx(HW_2X16))
- def test_a_model_too_big_for_one_card_and_ram_recommends_the_split(self):
+ def test_r_restores_the_recommendation(self):
async def case(app, pilot, ctx):
- await press(pilot, "enter", "enter")
- cur = app.screen.menu().current
- self.assertEqual(sorted(cur.key), [0, 1])
- hw = dict(HW_2X16)
- hw["ram_gb"] = 24.0
- self.run_app(case, ctx=make_ctx(hw))
+ await press(pilot, "enter", "enter", "down", "o", "r", "enter")
+ self.assertIsNone(ctx.settings.gpu_roles)
+ self.run_app(case, ctx=make_ctx(HW_2X16))
+
+ def test_every_card_off_is_allowed_with_one_short_line(self):
+ async def case(app, pilot, ctx):
+ await press(pilot, "enter", "enter", "down", "o", "up", "o")
+ flat = " ".join(text_of(app).split())
+ self.assertIn("! This engine needs a graphics card to start. Turn one on.", flat)
+ await press(pilot, "enter")
+ self.assertEqual(ctx.settings.gpu_roles, {"0": "off", "1": "off"}) # allowed; the server says plainly what the engine can do
+ self.run_app(case, ctx=make_ctx(HW_2X16))
+
+ def test_d_shows_what_the_roles_mean(self):
+ async def case(app, pilot, ctx):
+ await press(pilot, "enter", "enter", "d")
+ self.assertIn("Helper: holds a share of the model", " ".join(text_of(app).split()))
+ self.run_app(case, ctx=make_ctx(HW_2X16))
def test_an_integrated_chip_beside_one_card_gives_no_cards_step(self):
async def case(app, pilot, ctx):
diff --git a/tools/test_setup_golden.json b/tools/test_setup_golden.json
index 5580b3b7d..73725ecc7 100644
--- a/tools/test_setup_golden.json
+++ b/tools/test_setup_golden.json
@@ -295,9 +295,14 @@
"cwd": "<T>",
"exe": "<T>/engine/<EXE>",
"gpu": [
- 0,
- 1
+ 1,
+ 0
],
+ "gpu_order": "as_given",
+ "gpu_roles": {
+ "0": "main",
+ "1": "helper"
+ },
"gpus_asked": true,
"layer_split": "auto",
"lib_dirs": [
@@ -350,9 +355,14 @@
"cwd": "<T>",
"exe": "<T>/engine/<EXE>",
"gpu": [
- 0,
- 1
+ 1,
+ 0
],
+ "gpu_order": "as_given",
+ "gpu_roles": {
+ "0": "main",
+ "1": "helper"
+ },
"gpus_asked": true,
"layer_split": "auto",
"lib_dirs": [
@@ -404,8 +414,17 @@
],
"cwd": "<T>",
"exe": "<T>/engine/<EXE>",
- "gpu": 0,
+ "gpu": [
+ 1,
+ 0
+ ],
+ "gpu_order": "as_given",
+ "gpu_roles": {
+ "0": "main",
+ "1": "helper"
+ },
"gpus_asked": true,
+ "layer_split": "auto",
"lib_dirs": [
"<cuda>"
],
@@ -448,15 +467,24 @@
"131072",
"--kv",
"int8",
- "--mmap-experts",
+ "--resident-experts",
"--vision",
"--vram-reserve-mib",
"700"
],
"cwd": "<T>",
"exe": "<T>/engine/<EXE>",
- "gpu": 0,
+ "gpu": [
+ 1,
+ 0
+ ],
+ "gpu_order": "as_given",
+ "gpu_roles": {
+ "0": "main",
+ "1": "helper"
+ },
"gpus_asked": true,
+ "layer_split": "auto",
"lib_dirs": [
"<cuda>"
],
@@ -509,6 +537,10 @@
},
"exe": "<T>/engine/<EXE>",
"gpu": 0,
+ "gpu_roles": {
+ "0": "main",
+ "1": "off"
+ },
"gpus_asked": true,
"lib_dirs": [
"<cuda>"
@@ -558,9 +590,14 @@
"cwd": "<T>",
"exe": "<T>/engine/<EXE>",
"gpu": [
- 0,
- 1
+ 1,
+ 0
],
+ "gpu_order": "as_given",
+ "gpu_roles": {
+ "0": "main",
+ "1": "helper"
+ },
"gpus_asked": true,
"layer_split": "auto",
"lib_dirs": [
@@ -611,8 +648,17 @@
],
"cwd": "<T>",
"exe": "<T>/engine/<EXE>",
- "gpu": 0,
+ "gpu": [
+ 1,
+ 0
+ ],
+ "gpu_order": "as_given",
+ "gpu_roles": {
+ "0": "main",
+ "1": "helper"
+ },
"gpus_asked": true,
+ "layer_split": "auto",
"lib_dirs": [
"<cuda>"
],
@@ -660,8 +706,17 @@
],
"cwd": "<T>",
"exe": "<T>/engine/<EXE>",
- "gpu": 0,
+ "gpu": [
+ 1,
+ 0
+ ],
+ "gpu_order": "as_given",
+ "gpu_roles": {
+ "0": "main",
+ "1": "helper"
+ },
"gpus_asked": true,
+ "layer_split": "auto",
"lib_dirs": [
"<cuda>"
],
@@ -705,13 +760,22 @@
"65536",
"--kv",
"int8",
- "--mmap-experts",
+ "--resident-experts",
"--vision"
],
"cwd": "<T>",
"exe": "<T>/engine/<EXE>",
- "gpu": 0,
+ "gpu": [
+ 1,
+ 0
+ ],
+ "gpu_order": "as_given",
+ "gpu_roles": {
+ "0": "main",
+ "1": "helper"
+ },
"gpus_asked": true,
+ "layer_split": "auto",
"lib_dirs": [
"<cuda>"
],
@@ -763,6 +827,10 @@
},
"exe": "<T>/engine/<EXE>",
"gpu": 0,
+ "gpu_roles": {
+ "0": "main",
+ "1": "off"
+ },
"gpus_asked": true,
"lib_dirs": [
"<cuda>"
diff --git a/tools/test_setup_golden.py b/tools/test_setup_golden.py
index 01c712b84..2d5ea6186 100644
--- a/tools/test_setup_golden.py
+++ b/tools/test_setup_golden.py
@@ -98,6 +98,7 @@ def install(ram, found, argv, answers=None, extra=(), avx512=False, configs=()):
mock.patch.object(setup, "load_settings", lambda: {}),
mock.patch.object(setup, "save_settings", lambda s: None),
mock.patch.object(setup, "gpus", lambda: found),
+ mock.patch.object(setup, "card_speeds", lambda: {}),
mock.patch.object(setup, "amd_gpus", lambda: []),
mock.patch.object(setup, "ram_gb", lambda: ram),
mock.patch.object(setup, "cpu_info", lambda: ("Test CPU", True, avx512)),
@@ -179,7 +180,7 @@ class Golden(unittest.TestCase):
lowram_multi = self.golden["32GB-2x24GB qwen IQ3_XXS"]["config"] # #364 -> #642: low-RAM, two cards -> both
self.assertIn("--resident-experts" if "--resident-experts" in lowram_multi["args"] else "--mmap-experts",
lowram_multi["args"])
- self.assertEqual(lowram_multi.get("gpu"), [0, 1]) # 0.1.40 (#642): resident on both cards
+ self.assertEqual(lowram_multi.get("gpu"), [1, 0]) # 0.1.40 (#642): resident on both cards; roles: Main 0 goes last
def test_yes(self):
for key, ram, found, family, model in self.cases():
diff --git a/tools/test_setup_gpu_fit.py b/tools/test_setup_gpu_fit.py
index 237c9a240..20e8e1b3a 100644
--- a/tools/test_setup_gpu_fit.py
+++ b/tools/test_setup_gpu_fit.py
@@ -1,5 +1,5 @@
-"""The owner's multi-GPU rule in setup (strata_control/gpu_fit.py): the model goes on the LARGEST card + RAM first; the second card is
-offered as a layer split and is the default only when the model does not fit one card (with RAM) or would be too tight.
+"""Graphics card roles in setup (strata_control/gpu_roles.py): the best card is Main, the others start as Helper, a card with 8 GB or less
+starts Off; one question, "Use these roles?"; --gpu-roles names them.
python tools/test_setup_gpu_fit.py
"""
@@ -28,28 +28,42 @@ class SetupFitRule(unittest.TestCase):
self.assertEqual(cfg["gpu"], 0) if "gpu" in cfg else None
self.assertFalse(any("Which GPUs" in q for q in asked), asked)
- def test_two_unequal_cards_and_the_model_fits_one_defaults_to_the_largest_card_and_offers_the_split(self):
+ def test_two_unequal_cards_the_big_one_is_main_and_a_card_with_8_gb_or_less_starts_off(self):
found = [card(0, "NVIDIA GeForce RTX 3060 Ti", 8.0, "86"), card(1, "NVIDIA GeForce RTX 3070 Ti", 16.0, "86")]
code, out, cfg, asked = run(62.0, found, "IQ3_XXS")
self.assertEqual(code, 0, out)
self.assertEqual(cfg["gpu"], 1) # the 16 GB card, alone
+ self.assertEqual(cfg["gpu_roles"], {"0": "off", "1": "main"})
self.assertNotIn("layer_split", cfg)
- self.assertIn("layer split", out) # the split was offered
- self.assertIn("1) RTX 3070 Ti + RAM", out.replace("NVIDIA GeForce ", ""))
- self.assertTrue(any("Which GPUs" in q and "[1]" in q for q in asked), asked)
- # choosing the split is one answer away
- code, out, cfg, _ = run(62.0, found, "IQ3_XXS", answers={"Which GPUs?": "2"})
+ self.assertTrue(any("Use these roles?" in q and "[y]" in q for q in asked), asked)
+ # one answer away: the 8 GB card as a Helper
+ code, out, cfg, _ = run(62.0, found, "IQ3_XXS", answers={"Use these roles?": "n", "Roles, e.g.": "1=main,0=helper"})
self.assertEqual(code, 0, out)
- self.assertEqual(sorted(cfg["gpu"]), [0, 1])
+ self.assertEqual(cfg["gpu"], [0, 1]) # Main LAST: the engine's stage order
+ self.assertEqual(cfg["gpu_order"], "as_given")
+ self.assertEqual(cfg["gpu_roles"], {"0": "helper", "1": "main"})
self.assertEqual(cfg["layer_split"], "auto")
- def test_two_small_cards_and_the_model_does_not_fit_one_defaults_to_the_split(self):
+ def test_two_equal_cards_start_as_main_and_helper_with_the_lower_number_main(self):
found = [card(0, "NVIDIA GeForce RTX 3060", 12.0, "86"), card(1, "NVIDIA GeForce RTX 3060", 12.0, "86")]
code, out, cfg, asked = run(26.0, found, "IQ3_XXS")
self.assertEqual(code, 0, out)
- self.assertEqual(sorted(cfg["gpu"]), [0, 1])
+ self.assertEqual(cfg["gpu"], [1, 0])
+ self.assertEqual(cfg["gpu_roles"], {"0": "main", "1": "helper"})
+ self.assertEqual(cfg["gpu_order"], "as_given")
self.assertEqual(cfg["layer_split"], "auto")
- self.assertTrue(any("Which GPUs" in q and "[2]" in q for q in asked), asked)
+
+ def test_gpu_roles_flag(self):
+ found = [card(0, "NVIDIA GeForce RTX 3060", 12.0, "86"), card(1, "NVIDIA GeForce RTX 3070 Ti", 16.0, "86")]
+ code, out, cfg, _ = run(62.0, found, "IQ3_XXS", extra=("--gpu-roles", "0=main,1=helper"))
+ self.assertEqual(code, 0, out)
+ self.assertEqual(cfg["gpu"], [1, 0]) # the slower card is Main on purpose: it goes last
+ self.assertEqual(cfg["gpu_roles"], {"0": "main", "1": "helper"})
+ code, out, cfg, _ = run(62.0, found, "IQ3_XXS", extra=("--gpu-roles", "1=main,0=off"))
+ self.assertEqual((cfg["gpu"], cfg["gpu_roles"]), (1, {"0": "off", "1": "main"}))
+ self.assertNotIn("layer_split", cfg)
+ code, out, cfg, _ = run(62.0, found, "IQ3_XXS", extra=("--gpu-roles", "0=off,1=off"))
+ self.assertNotEqual(code, 0)
def test_naming_the_cards_still_wins(self):
found = [card(0, "NVIDIA GeForce RTX 3070 Ti", 16.0, "86"), card(1, "NVIDIA GeForce RTX 3060 Ti", 8.0, "86")]
diff --git a/tools/test_setup_risk.py b/tools/test_setup_risk.py
index be0cf34c9..b3562e2c6 100644
--- a/tools/test_setup_risk.py
+++ b/tools/test_setup_risk.py
@@ -238,10 +238,10 @@ class LowRamGpus(unittest.TestCase):
def test_asked_and_answered_two(self):
ram, found = PROFILES["32GB-2x24GB"]
code, out, cfg, asked = install(ram, found, ["--family", "qwen", "--model", "Q2_0", "--no-start"],
- answers={"Which GPUs?": "2", "Low-RAM mode: which GPUs?": "2"}) # the split asked for (the default is the largest card + RAM)
+ answers={"Low-RAM mode: which GPUs?": "2"}) # the roles are the default (Main last); the low-RAM question still defaults to one card
self.assertEqual(code, 0, out)
self.assertTrue(any("Low-RAM mode: which GPUs? [1]" in q for q in asked), asked)
- self.assertEqual(cfg["gpu"], [0, 1])
+ self.assertEqual(cfg["gpu"], [1, 0])
self.assertIn("--mmap-experts", cfg["args"])
self.assertIn("1) GPU 0 (NVIDIA GeForce RTX 3090, 24 GB) only", out)
@@ -276,9 +276,9 @@ class LowRamGpus(unittest.TestCase):
ram, found = PROFILES["32GB-2x24GB"]
with mock.patch.object(setup, "MIN_ENGINE", setup.RESIDENT_SPLIT_ENGINE):
code, out, cfg, asked = install(ram, found, ["--family", "qwen", "--model", "Q2_0", "--no-start"],
- answers={"Which GPUs?": "2"}) # the split chosen (the default is the largest card + RAM)
+ answers={"Use these roles?": "y"}) # the recommended roles: Main 0 (last), Helper 1
self.assertEqual(code, 0, out)
- self.assertEqual(cfg["gpu"], [0, 1])
+ self.assertEqual(cfg["gpu"], [1, 0])
self.assertIn("--resident-experts", cfg["args"])
self.assertFalse(any("Low-RAM mode" in q for q in asked), asked)
self.assertIn("stay in RAM", out)
@@ -401,11 +401,12 @@ class SplitShortCard(unittest.TestCase):
eights = [card(i, "NVIDIA GeForce RTX 3070", 8.0, "86") for i in range(2)]
self.assertEqual(setup.split_short(eights), []) # neither card alone reads more
- def test_yes_takes_the_first_card(self):
+ def test_yes_takes_the_recommended_roles(self):
+ """The 10 GB card is above the 8 GB line, so it starts as a Helper (the owner's rule); the first card is Main and goes last."""
code, out, cfg, _ = install(127.8, self.FOUND, ["--family", "qwen", "--model", "Q2_0", "--no-start"])
self.assertEqual(code, 0, out)
- self.assertEqual(cfg["gpu"], 0)
- self.assertIn("too small to lend a split its prompt buffers", out)
+ self.assertEqual(cfg["gpu"], [1, 0])
+ self.assertEqual(cfg["gpu_roles"], {"0": "main", "1": "helper"})
def test_named_pair_is_kept(self):
code, out, cfg, _ = install(127.8, self.FOUND, ["--family", "qwen", "--model", "Q2_0", "--no-start",
diff --git a/tools/test_setup_unsloth.py b/tools/test_setup_unsloth.py
index 1fc970705..f6cc3d180 100644
--- a/tools/test_setup_unsloth.py
+++ b/tools/test_setup_unsloth.py
@@ -352,7 +352,7 @@ class Main(Base):
"""#967: images are allowed with UD-Q4_K_XL (tested end to end since MODEL-UPDATE)."""
code, out, cfg = self.main(["--context", "8192", "--vision", "yes", "--low-ram", "on"], n_gpus=2)
self.assertEqual(code, 0, out)
- self.assertIn("(NVIDIA GeForce RTX 5070, 12 GB) only (--gpus 0,1 shares it across", out)
+ self.assertIn("(NVIDIA GeForce RTX 5070, 12 GB) only (--gpus 1,0 shares it across", out)
self.assertNotIn("images are not available", out)
self.assertIn("images: on", out)
self.assertNotIn("are untested", out) # MODEL-UPDATE: run end to end (CUDA, HIP)
@@ -385,8 +385,7 @@ class Main(Base):
self.assertAlmostEqual(need, setup.MODELS[M]["download_gb"] + setup.UNSLOTH_RAM_LEFT_GB)
code, out, cfg = self.main(["--context", "8192"], n_gpus=2, ram=127.8) # no --gpus: stays on one GPU
self.assertEqual(code, 0, out)
- self.assertNotIn("uses them together anyway", out) # one card + RAM is the default now: nothing to warn about
- self.assertNotIn("layer_split", cfg)
+ self.assertNotIn("layer_split", cfg) # the model cannot share its RAM budget: one card (the Main one)
self.assertEqual(cfg["args"][cfg["args"].index("--resident-budget-gib") + 1], "71") # all of them
code, out, cfg = self.main(["--context", "8192", "--gpus", "0,1"], n_gpus=2, ram=127.8) # asked for by name
self.assertEqual(code, 0, out)
@@ -434,15 +433,14 @@ class Main(Base):
self.assertIn("--resident-budget-gib", cfg["args"])
code, out, cfg = self.main(["--context", "8192"], n_gpus=2, answers={"which GPUs?": "2"})
self.assertEqual(code, 0, out)
- self.assertEqual(cfg["gpu"], [0, 1])
+ self.assertEqual(cfg["gpu"], [1, 0])
self.assertIn("--resident-budget-gib", cfg["args"])
@budget_mode()
def test_yes_alone_keeps_one_gpu(self):
code, out, cfg = self.main(["--context", "8192"], n_gpus=2, ram=165.0)
self.assertEqual(code, 0, out)
- self.assertIn("layer split", out) # the second card is offered as a split
- self.assertIn("(recommended)", out)
+ self.assertIn("Main: GPU 1", out) # the roles are shown; the model keeps to one card
self.assertNotIn("layer_split", cfg)
self.assertIn("--resident-budget-gib", cfg["args"])
@@ -644,12 +642,12 @@ class LayerSplit(unittest.TestCase):
argv = ["--family", "unsloth", "--model", M, "--context", "32768", "--no-start"]
code, out, cfg, asked = install(165.0, self.found, argv, answers={"which GPUs?": "1", "Which GPUs?": "1"})
self.assertEqual(code, 0, out)
- self.assertTrue(any("Which GPUs? [1]" in q for q in asked), asked) # the fit rule's question: the largest card + RAM is the default
+ self.assertTrue(any("Use these roles? [y]" in q for q in asked), asked) # the roles question; the model's own question follows
self.assertEqual(cfg["gpu"], 0)
self.assertIn("--resident-budget-gib", cfg["args"])
- code, out, cfg, asked = install(165.0, self.found, argv, answers={"Which GPUs?": "2", f"{M}: which GPUs?": "2"})
+ code, out, cfg, asked = install(165.0, self.found, argv, answers={f"{M}: which GPUs?": "2"})
self.assertEqual(code, 0, out)
- self.assertEqual(cfg["gpu"], [0, 1])
+ self.assertEqual(cfg["gpu"], [1, 0])
self.assertEqual(cfg["layer_split"], "auto")
self.assertNotIn("--resident-budget-gib", cfg["args"])
@@ -806,9 +804,9 @@ class Resident(Base):
def test_two_gpus_split_with_every_expert_in_ram(self):
"""Where the experts fit, two cards are the GGUF models' layer split (no budget question, no #498 note)."""
- code, out, cfg = self.main(["--context", "8192"], n_gpus=2, ram=127.8, answers={"Which GPUs?": "2"}) # the split chosen
+ code, out, cfg = self.main(["--context", "8192"], n_gpus=2, ram=127.8, answers={"Use these roles?": "y"}) # the recommended roles
self.assertEqual(code, 0, out)
- self.assertEqual(cfg["gpu"], [0, 1])
+ self.assertEqual(cfg["gpu"], [1, 0])
self.assertEqual(cfg["layer_split"], "auto")
self.assertNotIn("--resident-budget-gib", cfg["args"])
self.assertNotIn("which GPUs?", out)