Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Keep the card a person chose: strata launch no longer re-picks GPU 0, the engine's GPU 0 is explained, old hip_ordinal and list GPUs for images

Niko12219bb1dd20444f
diff --git a/apps/host/server/native-control.mjs b/apps/host/server/native-control.mjs
index 7ec4a440d..3445a1380 100644
--- a/apps/host/server/native-control.mjs
+++ b/apps/host/server/native-control.mjs
@@ -208,7 +208,7 @@ return {summary:result.summary??'Deleted',removed:result.plan?.gb??null};}
const cfg={...config,...visionPreferences,...(resources?{...resources,ui_resource_profile:args.resourceProfile}:{}),host:'127.0.0.1',open_browser:false,...validateNativeSettings(args.values??{}),vision};
/* The app's suggested cards only fill a gap: a model whose settings file already names its GPUs ("gpu": [1, 0], with "gpu_order": "as_given") keeps them (#3256). A card the person picked in the screen always wins. */
const namedInFile=Array.isArray(config.gpu)?config.gpu.length>0:config.gpu!==undefined&&config.gpu!==null&&config.gpu!=='';
- if(args.gpus!==undefined&&!(args.gpusFromRecommendation&&namedInFile)){const detected=new Set((snapshot.hardware?.gpus??[]).map(g=>g.index));if(!Array.isArray(args.gpus)||!args.gpus.length||new Set(args.gpus).size!==args.gpus.length||args.gpus.some(g=>!Number.isInteger(g)||!detected.has(g)))throw new Error('Choose distinct detected GPUs. CPU-only text inference is not supported by this native adapter.');cfg.gpu=args.gpus.length===1?args.gpus[0]:args.gpus;cfg.layer_split=args.gpus.length>1?validateNativeSettings({layer_split:args.layerSplit??'auto'}).layer_split:'auto';if(cfg.layer_split!=='auto'&&cfg.layer_split.split(',').length!==args.gpus.length-1)throw new Error('Choose one layer split point between each pair of GPUs.');}
+ if(args.gpus!==undefined&&!(args.gpusFromRecommendation&&namedInFile)){const detected=new Set((snapshot.hardware?.gpus??[]).map(g=>g.index));if(!Array.isArray(args.gpus)||!args.gpus.length||new Set(args.gpus).size!==args.gpus.length||args.gpus.some(g=>!Number.isInteger(g)||!detected.has(g)))throw new Error('Choose distinct detected GPUs. CPU-only text inference is not supported by this native adapter.');cfg.gpu=args.gpus.length===1?args.gpus[0]:args.gpus;delete cfg.hip_ordinal;cfg.layer_split=args.gpus.length>1?validateNativeSettings({layer_split:args.layerSplit??'auto'}).layer_split:'auto';if(cfg.layer_split!=='auto'&&cfg.layer_split.split(',').length!==args.gpus.length-1)throw new Error('Choose one layer split point between each pair of GPUs.');}
if(cfg.vram_reserve_mib!==undefined&&cfg.vram_reserve_mib!==null){cfg.args=[...(cfg.args??[])];const at=cfg.args.indexOf('--vram-reserve-mib');if(at>=0)cfg.args.splice(at,2);cfg.args.push('--vram-reserve-mib',String(cfg.vram_reserve_mib));}
if(mine.length){cfg.args=mergeUserArgs(cfg.args??[],mine);cfg[USER_KEY]=mine;}else delete cfg[USER_KEY];
if(cfg.vision){if(cfg.vision_tokens!=null)cfg.vision={...cfg.vision,max_tokens:cfg.vision_tokens};if(cfg.vision_min_tokens!=null)cfg.vision={...cfg.vision,min_tokens:cfg.vision_min_tokens};}cfg.mcp={...(cfg.mcp??{}),...(cfg.mcp_timeout_s!=null?{timeout_s:cfg.mcp_timeout_s}:{}),...(cfg.mcp_max_rounds!=null?{max_rounds:cfg.mcp_max_rounds}:{}),...(cfg.mcp_max_result_chars!=null?{max_result_chars:cfg.mcp_max_result_chars}:{})};
diff --git a/apps/host/tests/native-gpu-choice.test.mjs b/apps/host/tests/native-gpu-choice.test.mjs
index b209cdcb0..2be9cdbed 100644
--- a/apps/host/tests/native-gpu-choice.test.mjs
+++ b/apps/host/tests/native-gpu-choice.test.mjs
@@ -96,3 +96,22 @@ test('#3211 #3221: the Monitor opens on the model\'s assigned cards and the drop
const css = await readFile(new URL('../src/styles/annotation-fixes.css', import.meta.url), 'utf8');
assert.match(css, /\.select__opt \{ height: auto; min-height: 36px;/, 'a wrapped name no longer runs over the next row');
});
+
+test('a card picked in the screen replaces an old AMD config\'s "hip_ordinal" (it would send the engine to the old card)', async () => {
+ const f = nativeFixture();
+ try {
+ await f.control.refresh(true);
+ const { readFileSync, writeFileSync } = await import('node:fs');
+ const { join } = await import('node:path');
+ const path = join(f.repo, 'strata-iq2_xs.json');
+ const config = JSON.parse(readFileSync(path, 'utf8')); config.hip_ordinal = 0; writeFileSync(path, JSON.stringify(config));
+ await f.control.refresh(true);
+ const done = await finishJob(f.control, f.control.operation('load', { model: 'iq2_xs', gpus: [1] }).id);
+ assert.equal(done.state, 'done', done.error);
+ const instance = f.control.resolveInstance(done.result.instanceId);
+ assert.deepEqual(instance.gpus, [1]);
+ const written = JSON.parse(readFileSync(instance.configPath, 'utf8'));
+ assert.equal(written.gpu, 1);
+ assert.equal('hip_ordinal' in written, false);
+ } finally { await f.cleanup(); }
+});
diff --git a/serve/images.py b/serve/images.py
index 9a7260e3d..a8a06cc7b 100644
--- a/serve/images.py
+++ b/serve/images.py
@@ -141,7 +141,8 @@ def image_env(cfg: dict, base: dict | None = None) -> dict:
var = {"cuda": "CUDA_VISIBLE_DEVICES", "hip": "HIP_VISIBLE_DEVICES",
"vulkan": "GGML_VK_VISIBLE_DEVICES"}.get(cfg.get("backend"))
if var:
- env[var] = str(gpu)
+ # one card is "1"; a list ([1, 0], "1,0") is every one of them, in order - str() of a list is not a device list
+ env[var] = ",".join(str(x).strip() for x in gpu if str(x).strip() != "") if isinstance(gpu, (list, tuple)) else str(gpu)
if var == "CUDA_VISIBLE_DEVICES":
env["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
for k, v in (cfg.get("env") or {}).items():
diff --git a/serve/server.py b/serve/server.py
index 0635a715f..1cfadc6af 100644
--- a/serve/server.py
+++ b/serve/server.py
@@ -3293,6 +3293,22 @@ def hip_env_guard(env: dict, cfg: dict) -> list[str]:
return said
+def card_note(cfg: dict, env: dict) -> str | None:
+ """One plain line that ties the engine's own numbering to the person's: the engine counts the cards it is given from 0 ("GPU 0",
+ "CUDA0" in its log), so a person who chose GPU 1 would read "GPU 0" and think the choice was lost. Said only when the two differ
+ (NVIDIA, numbered as nvidia-smi numbers them); None otherwise."""
+ if cfg.get("backend") == "hip" or not gpu_list(cfg):
+ return None
+ mine = [x.strip() for x in str((env or {}).get("CUDA_VISIBLE_DEVICES", "")).split(",") if x.strip() != ""]
+ if not mine or mine == [str(i) for i in range(len(mine))]:
+ return None
+ names = [f"CUDA{i}" for i in range(len(mine))]
+ if len(mine) == 1:
+ return f"[strata] running on GPU {mine[0]} (as nvidia-smi numbers it); the engine's log calls it GPU 0 / CUDA0"
+ return (f"[strata] cards in stage order: GPU {', then GPU '.join(mine)} (as nvidia-smi numbers them); the engine's log calls them "
+ f"{', '.join(names)} - the first runs the first layers and the picture reader, the last runs the head and the draft layer")
+
+
def vision_env(cfg: dict, env: dict) -> dict:
"""The image encoder's environment: the engine's, unless the config's vision section names its own "cuda_device"
(numbered like nvidia-smi) - then the encoder runs on that card alone, so a spare GPU can hold it while the engine
@@ -7958,6 +7974,9 @@ def main(argv=None, *, lifecycle_event=None, interrupt_exit_code: int = 0) -> in
print("model unloaded; the first request loads it ..." if lazy else
"loading the model (the first start takes a minute or two) ...", flush=True)
warn_budget_over_ram(engine_args(cfg) if "args" in cfg else []) # #1080
+ note = card_note(cfg, env)
+ if note:
+ print(note, flush=True)
if layer_split_of(cfg):
try:
split = layer_split_value(cfg) # #644: before the (minutes-long) start
diff --git a/serve/test_images.py b/serve/test_images.py
index 2842e5230..95ad7fbe9 100644
--- a/serve/test_images.py
+++ b/serve/test_images.py
@@ -96,6 +96,7 @@ class Command(unittest.TestCase):
def test_env(self):
env = images_api.image_env({"backend": "cuda", "gpu": 1, "env": {"X": 2}}, {"PATH": "p"})
self.assertEqual((env["CUDA_VISIBLE_DEVICES"], env["CUDA_DEVICE_ORDER"], env["X"]), ("1", "PCI_BUS_ID", "2"))
+ self.assertEqual(images_api.image_env({"backend": "cuda", "gpu": [1, 0]}, {})["CUDA_VISIBLE_DEVICES"], "1,0") # a list is a device list
with tempfile.TemporaryDirectory() as d:
env = images_api.image_env({"backend": "hip", "gpu": 0, "lib_dirs": [d, d + "-missing"]}, {})
self.assertEqual(env["HIP_VISIBLE_DEVICES"], "0")
diff --git a/serve/test_server.py b/serve/test_server.py
index 8bbe6e65b..4249585d4 100644
--- a/serve/test_server.py
+++ b/serve/test_server.py
@@ -1726,6 +1726,19 @@ class GpuChoice(unittest.TestCase):
self.assertEqual(plain.get("CUDA_VISIBLE_DEVICES"), os.environ.get("CUDA_VISIBLE_DEVICES"))
self.assertEqual(plain.get("CUDA_DEVICE_ORDER"), os.environ.get("CUDA_DEVICE_ORDER"))
+ def test_card_note_ties_the_engines_numbering_to_the_persons(self):
+ """The engine counts the cards it is given from 0, so GPU 1 reads "GPU 0 / CUDA0" in its log: one line says which is which."""
+ from serve.server import card_note, child_env
+ one = {"gpu": 1, "args": []}
+ self.assertIn("GPU 1", card_note(one, child_env(one)))
+ self.assertIn("GPU 0 / CUDA0", card_note(one, child_env(one)))
+ both = {"gpu": [1, 0], "args": [], "gpu_order": "as_given"}
+ line = card_note(both, child_env(both))
+ self.assertIn("GPU 1, then GPU 0", line)
+ self.assertIn("CUDA0, CUDA1", line)
+ for same in ({"gpu": 0, "args": []}, {"gpu": [0, 1], "args": [], "gpu_order": "as_given"}, {"args": []}, {"gpu": 1, "backend": "hip"}):
+ self.assertIsNone(card_note(same, child_env(same)), same) # the numbers agree, or no card was chosen, or AMD (its own numbering)
+
def test_card_order(self):
"""#1352: with an auto layer split the faster card (SMs x clock) goes last; ties, a manual split, a
measurement we lack, HIP and "gpu_order": "as_given" keep the config's order."""
diff --git a/strata_cli/launch/app.py b/strata_cli/launch/app.py
index 8a327b95a..29d575603 100644
--- a/strata_cli/launch/app.py
+++ b/strata_cli/launch/app.py
@@ -174,7 +174,7 @@ class Ctx:
from serve.server import engine_args
import contextlib
import io
- d = derive_config(apply_choices(cfg, s), s, bool_flags=tune_knobs.BOOL_FLAGS, gpus=cards_for_launch(s, self.catalog))
+ d = derive_config(apply_choices(cfg, s), s, bool_flags=tune_knobs.BOOL_FLAGS, gpus=cards_for_launch(s, self.catalog, base=cfg))
if s.parallel:
d["parallel"] = s.parallel
with contextlib.redirect_stdout(io.StringIO()):
diff --git a/strata_cli/launch/runner.py b/strata_cli/launch/runner.py
index d8c223641..688459800 100644
--- a/strata_cli/launch/runner.py
+++ b/strata_cli/launch/runner.py
@@ -214,17 +214,36 @@ def apply_cards(cfg: dict, gpus: list[int] | None) -> None:
gpus = [int(g) for g in gpus]
cfg["gpu"] = gpus[0] if len(gpus) == 1 else gpus
cfg["gpus_asked"] = True
+ cfg.pop("hip_ordinal", None) # an old AMD config's ordinal of ITS card would win over the card named now
if len(gpus) > 1:
cfg["layer_split"] = cfg.get("layer_split") or "auto"
else:
cfg.pop("layer_split", None)
-def cards_for_launch(s: S.LaunchSettings, catalog, ram_gb: float | None = None) -> list[int] | None:
- """The cards this launch uses: the ones named (--gpu / --gpus, the Cards step), else the recommended ones by the shared fit rule
- (strata_control/gpu_fit.py) when the computer has more than one card that counts, else None (one card: the installed config's own)."""
+def installed_cards(base: dict | None) -> list[int] | None:
+ """The cards the installed model's own config names after a choice (setup --gpu / --gpus, the app's card pick: "gpus_asked"), in its order;
+ None when it names none or was never asked."""
+ if not isinstance(base, dict) or not base.get("gpus_asked"):
+ return None
+ g = base.get("gpu")
+ if g is None or g == "":
+ return None
+ try:
+ cards = [int(str(x).strip()) for x in (g if isinstance(g, (list, tuple)) else str(g).split(",")) if str(x).strip() != ""]
+ except ValueError:
+ return None
+ return cards or None
+
+
+def cards_for_launch(s: S.LaunchSettings, catalog, ram_gb: float | None = None, base: dict | None = None) -> list[int] | None:
+ """The cards this launch uses: the ones named (--gpu / --gpus, the Cards step), else the cards the installed config was set to (a person's
+ earlier choice, "gpu": 1 or [1, 0] - never replaced by the automatic pick, which would put the first of equal cards back), else the
+ recommended ones by the shared fit rule (strata_control/gpu_fit.py) when the computer has more than one card that counts, else None."""
if s.gpus:
return [int(g) for g in s.gpus]
+ if installed_cards(base):
+ return None
try:
from strata_control import gpu_fit
comp = catalog.computer()
@@ -854,7 +873,7 @@ class Runner:
from strata_control import tune_knobs
base = read_config(base_path)
cfg = derive_config(base, self.s, bool_flags=tune_knobs.BOOL_FLAGS,
- log=str(launch_dir() / f"{self.name}.engine.log"), gpus=cards_for_launch(self.s, self.catalog))
+ log=str(launch_dir() / f"{self.name}.engine.log"), gpus=cards_for_launch(self.s, self.catalog, base=base))
out = launch_dir() / f"{self.name}.json"
write_private(out, cfg)
self.config_path = out
diff --git a/strata_cli/launch/screens.py b/strata_cli/launch/screens.py
index 537309373..917df146e 100644
--- a/strata_cli/launch/screens.py
+++ b/strata_cli/launch/screens.py
@@ -544,6 +544,14 @@ class CardsScreen(Base):
r = next((x for x in self.ctx.rows if (x.family, x.model) == (s.family, s.model)), None)
return float(r.memory_gb) if r is not None and r.memory_gb else 44.0
+ def installed(self) -> list[int] | None:
+ """The cards the installed model was set to by a person's choice (kept unless this launch names others)."""
+ from strata_cli.launch.runner import installed_cards
+ try:
+ return installed_cards(self.ctx.base_config())
+ except Exception: # noqa: BLE001
+ return None
+
def choices(self) -> dict:
from strata_control import gpu_fit
return gpu_fit.options(self.model_memory(), self.ctx.computer.ram_gb, self.ctx.computer.gpus, self.ctx.computer.engine_vendor)
@@ -553,10 +561,11 @@ class CardsScreen(Base):
items = []
sel = 0
cur = self.ctx.settings.gpus
+ inst = self.installed()
for i, o in enumerate(self.res["options"]):
- label = o["label"] + (" (recommended)" if o["recommended"] else "")
+ label = o["label"] + (" (recommended)" if o["recommended"] else "") + (" (as installed)" if inst and sorted(o["cards"]) == sorted(inst) else "")
items.append(Item([label, Cell(o["words"], WARN if o["verdict"] in ("spills", "tight") else "")], key=o["cards"]))
- if cur is None and o["recommended"] or (cur is not None and o["cards"] == list(cur)):
+ if cur is None and (inst is None and o["recommended"] or inst is not None and sorted(o["cards"]) == sorted(inst)) or (cur is not None and o["cards"] == list(cur)):
sel = i
wide = max((len(str(i.cells[0])) for i in items), default=40)
self.query_one(Menu).columns = [("", max(40, min(wide + 3, 70))), ("", 0)]
@@ -587,7 +596,9 @@ class CardsScreen(Base):
def commit(self) -> None:
cur = self.menu().current
rec = next((o["cards"] for o in self.res["options"] if o["recommended"]), None)
- self.ctx.settings.gpus = None if cur.key == rec else list(cur.key) # the recommended choice stays "let Strata choose" (the same rule in setup)
+ inst = self.installed()
+ keep = rec if inst is None else inst # "let Strata choose" = the recommendation, or the cards the installed model was set to
+ self.ctx.settings.gpus = None if keep is not None and sorted(cur.key) == sorted(keep) else list(cur.key) # the recommended choice stays "let Strata choose" (the same rule in setup)
# ------------------------------------------------------------------------------------------------ 3 folder
diff --git a/tests/terminal/test_launch.py b/tests/terminal/test_launch.py
index 18fd91bd8..e471c9612 100644
--- a/tests/terminal/test_launch.py
+++ b/tests/terminal/test_launch.py
@@ -580,6 +580,8 @@ class AdoptRules(unittest.TestCase):
cfg = derive_config(split_base, S.LaunchSettings(), bool_flags=tune_knobs.BOOL_FLAGS)
self.assertEqual(cfg["gpu"], [0, 1])
self.assertEqual(base["gpu"], 0) # the installed config itself is never changed
+ aged = derive_config({**base, "backend": "hip", "hip_ordinal": 1}, S.LaunchSettings(), bool_flags=tune_knobs.BOOL_FLAGS, gpus=[0])
+ self.assertNotIn("hip_ordinal", aged) # a card named now is not overruled by an old config's ordinal
# which cards: named ones win (one card too: --gpu 1 on a box that has them), else the fit rule on a box with several cards
two = catalog(HW_2X16)
self.assertEqual(cards_for_launch(S.LaunchSettings(family="qwen", model="IQ3_XXS", gpus=[1, 0]), two), [1, 0])
@@ -596,6 +598,34 @@ class AdoptRules(unittest.TestCase):
self.assertEqual(json.loads(out.read_text())["gpu"], 1)
self.assertEqual(json.loads(installed.read_text())["gpu"], 0)
+ def test_a_card_the_installed_model_was_set_to_is_not_replaced_by_the_automatic_pick(self):
+ """Two equal cards: the fit rule recommends card 0, so a person's earlier choice (setup --gpu 1, the app's card pick: "gpus_asked")
+ must survive a `strata launch` that names no cards - in its order, with "gpu_order": "as_given" too."""
+ import json
+ from tests.terminal.launch_support import HW_2X16
+ from strata_cli.launch.runner import cards_for_launch, installed_cards
+ with Home() as h:
+ two = catalog(HW_2X16)
+ plain = S.LaunchSettings(family="qwen", model="IQ3_XXS")
+ chosen = {"exe": "component:strata-engine", "args": ["--max-context", "32768"], "gpu": 1, "gpus_asked": True, "port": 8080}
+ self.assertEqual(installed_cards(chosen), [1])
+ self.assertEqual(installed_cards({**chosen, "gpu": [1, 0], "gpu_order": "as_given"}), [1, 0])
+ self.assertEqual(installed_cards({**chosen, "gpu": "1,0"}), [1, 0])
+ self.assertIsNone(installed_cards({**chosen, "gpus_asked": False})) # never asked: setup's default, not a choice
+ self.assertIsNone(installed_cards({"gpu": 1}))
+ self.assertIsNone(cards_for_launch(plain, two, base=chosen)) # the installed choice stays
+ self.assertEqual(cards_for_launch(plain, two, base={**chosen, "gpus_asked": False}), [0]) # no choice recorded: the fit rule
+ self.assertEqual(cards_for_launch(S.LaunchSettings(family="qwen", model="IQ3_XXS", gpus=[0]), two, base=chosen), [0]) # --gpu 0 wins
+ if True:
+ installed = h.base / "strata-iq3_xxs.json"
+ for gpu, extra in ((1, {}), ([1, 0], {"gpu_order": "as_given", "layer_split": "auto"})):
+ base = {**chosen, "gpu": gpu, **extra}
+ installed.write_text(json.dumps(base))
+ r = Runner(S.LaunchSettings(family="qwen", model="IQ3_XXS", name="keep"), catalog=two)
+ cfg = json.loads(r.build_config(installed).read_text())
+ self.assertEqual(cfg["gpu"], gpu)
+ self.assertEqual(cfg.get("gpu_order"), extra.get("gpu_order"))
+
def test_pictures_asked_but_still_off_is_said_in_plain_words(self):
with Home() as h:
cat = catalog(HW_12)