Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Keep the card a person chose: strata launch no longer re-picks GPU 0, the engine's GPU 0 is explained, old hip_ordinal and list GPUs for images
Niko12219bb1dd20444f
diff --git a/apps/host/server/native-control.mjs b/apps/host/server/native-control.mjsindex 7ec4a440d..3445a1380 100644--- a/apps/host/server/native-control.mjs+++ b/apps/host/server/native-control.mjs@@ -208,7 +208,7 @@ return {summary:result.summary??'Deleted',removed:result.plan?.gb??null};}const cfg={...config,...visionPreferences,...(resources?{...resources,ui_resource_profile:args.resourceProfile}:{}),host:'127.0.0.1',open_browser:false,...validateNativeSettings(args.values??{}),vision};/* The app's suggested cards only fill a gap: a model whose settings file already names its GPUs ("gpu": [1, 0], with "gpu_order": "as_given") keeps them (#3256). A card the person picked in the screen always wins. */const namedInFile=Array.isArray(config.gpu)?config.gpu.length>0:config.gpu!==undefined&&config.gpu!==null&&config.gpu!=='';- if(args.gpus!==undefined&&!(args.gpusFromRecommendation&&namedInFile)){const detected=new Set((snapshot.hardware?.gpus??[]).map(g=>g.index));if(!Array.isArray(args.gpus)||!args.gpus.length||new Set(args.gpus).size!==args.gpus.length||args.gpus.some(g=>!Number.isInteger(g)||!detected.has(g)))throw new Error('Choose distinct detected GPUs. CPU-only text inference is not supported by this native adapter.');cfg.gpu=args.gpus.length===1?args.gpus[0]:args.gpus;cfg.layer_split=args.gpus.length>1?validateNativeSettings({layer_split:args.layerSplit??'auto'}).layer_split:'auto';if(cfg.layer_split!=='auto'&&cfg.layer_split.split(',').length!==args.gpus.length-1)throw new Error('Choose one layer split point between each pair of GPUs.');}+ if(args.gpus!==undefined&&!(args.gpusFromRecommendation&&namedInFile)){const detected=new Set((snapshot.hardware?.gpus??[]).map(g=>g.index));if(!Array.isArray(args.gpus)||!args.gpus.length||new Set(args.gpus).size!==args.gpus.length||args.gpus.some(g=>!Number.isInteger(g)||!detected.has(g)))throw new Error('Choose distinct detected GPUs. CPU-only text inference is not supported by this native adapter.');cfg.gpu=args.gpus.length===1?args.gpus[0]:args.gpus;delete cfg.hip_ordinal;cfg.layer_split=args.gpus.length>1?validateNativeSettings({layer_split:args.layerSplit??'auto'}).layer_split:'auto';if(cfg.layer_split!=='auto'&&cfg.layer_split.split(',').length!==args.gpus.length-1)throw new Error('Choose one layer split point between each pair of GPUs.');}if(cfg.vram_reserve_mib!==undefined&&cfg.vram_reserve_mib!==null){cfg.args=[...(cfg.args??[])];const at=cfg.args.indexOf('--vram-reserve-mib');if(at>=0)cfg.args.splice(at,2);cfg.args.push('--vram-reserve-mib',String(cfg.vram_reserve_mib));}if(mine.length){cfg.args=mergeUserArgs(cfg.args??[],mine);cfg[USER_KEY]=mine;}else delete cfg[USER_KEY];if(cfg.vision){if(cfg.vision_tokens!=null)cfg.vision={...cfg.vision,max_tokens:cfg.vision_tokens};if(cfg.vision_min_tokens!=null)cfg.vision={...cfg.vision,min_tokens:cfg.vision_min_tokens};}cfg.mcp={...(cfg.mcp??{}),...(cfg.mcp_timeout_s!=null?{timeout_s:cfg.mcp_timeout_s}:{}),...(cfg.mcp_max_rounds!=null?{max_rounds:cfg.mcp_max_rounds}:{}),...(cfg.mcp_max_result_chars!=null?{max_result_chars:cfg.mcp_max_result_chars}:{})};diff --git a/apps/host/tests/native-gpu-choice.test.mjs b/apps/host/tests/native-gpu-choice.test.mjsindex b209cdcb0..2be9cdbed 100644--- a/apps/host/tests/native-gpu-choice.test.mjs+++ b/apps/host/tests/native-gpu-choice.test.mjs@@ -96,3 +96,22 @@ test('#3211 #3221: the Monitor opens on the model\'s assigned cards and the dropconst css = await readFile(new URL('../src/styles/annotation-fixes.css', import.meta.url), 'utf8');assert.match(css, /\.select__opt \{ height: auto; min-height: 36px;/, 'a wrapped name no longer runs over the next row');});++test('a card picked in the screen replaces an old AMD config\'s "hip_ordinal" (it would send the engine to the old card)', async () => {+ const f = nativeFixture();+ try {+ await f.control.refresh(true);+ const { readFileSync, writeFileSync } = await import('node:fs');+ const { join } = await import('node:path');+ const path = join(f.repo, 'strata-iq2_xs.json');+ const config = JSON.parse(readFileSync(path, 'utf8')); config.hip_ordinal = 0; writeFileSync(path, JSON.stringify(config));+ await f.control.refresh(true);+ const done = await finishJob(f.control, f.control.operation('load', { model: 'iq2_xs', gpus: [1] }).id);+ assert.equal(done.state, 'done', done.error);+ const instance = f.control.resolveInstance(done.result.instanceId);+ assert.deepEqual(instance.gpus, [1]);+ const written = JSON.parse(readFileSync(instance.configPath, 'utf8'));+ assert.equal(written.gpu, 1);+ assert.equal('hip_ordinal' in written, false);+ } finally { await f.cleanup(); }+});diff --git a/serve/images.py b/serve/images.pyindex 9a7260e3d..a8a06cc7b 100644--- a/serve/images.py+++ b/serve/images.py@@ -141,7 +141,8 @@ def image_env(cfg: dict, base: dict | None = None) -> dict:var = {"cuda": "CUDA_VISIBLE_DEVICES", "hip": "HIP_VISIBLE_DEVICES","vulkan": "GGML_VK_VISIBLE_DEVICES"}.get(cfg.get("backend"))if var:- env[var] = str(gpu)+ # one card is "1"; a list ([1, 0], "1,0") is every one of them, in order - str() of a list is not a device list+ env[var] = ",".join(str(x).strip() for x in gpu if str(x).strip() != "") if isinstance(gpu, (list, tuple)) else str(gpu)if var == "CUDA_VISIBLE_DEVICES":env["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"for k, v in (cfg.get("env") or {}).items():diff --git a/serve/server.py b/serve/server.pyindex 0635a715f..1cfadc6af 100644--- a/serve/server.py+++ b/serve/server.py@@ -3293,6 +3293,22 @@ def hip_env_guard(env: dict, cfg: dict) -> list[str]:return said+def card_note(cfg: dict, env: dict) -> str | None:+ """One plain line that ties the engine's own numbering to the person's: the engine counts the cards it is given from 0 ("GPU 0",+ "CUDA0" in its log), so a person who chose GPU 1 would read "GPU 0" and think the choice was lost. Said only when the two differ+ (NVIDIA, numbered as nvidia-smi numbers them); None otherwise."""+ if cfg.get("backend") == "hip" or not gpu_list(cfg):+ return None+ mine = [x.strip() for x in str((env or {}).get("CUDA_VISIBLE_DEVICES", "")).split(",") if x.strip() != ""]+ if not mine or mine == [str(i) for i in range(len(mine))]:+ return None+ names = [f"CUDA{i}" for i in range(len(mine))]+ if len(mine) == 1:+ return f"[strata] running on GPU {mine[0]} (as nvidia-smi numbers it); the engine's log calls it GPU 0 / CUDA0"+ return (f"[strata] cards in stage order: GPU {', then GPU '.join(mine)} (as nvidia-smi numbers them); the engine's log calls them "+ f"{', '.join(names)} - the first runs the first layers and the picture reader, the last runs the head and the draft layer")++def vision_env(cfg: dict, env: dict) -> dict:"""The image encoder's environment: the engine's, unless the config's vision section names its own "cuda_device"(numbered like nvidia-smi) - then the encoder runs on that card alone, so a spare GPU can hold it while the engine@@ -7958,6 +7974,9 @@ def main(argv=None, *, lifecycle_event=None, interrupt_exit_code: int = 0) -> inprint("model unloaded; the first request loads it ..." if lazy else"loading the model (the first start takes a minute or two) ...", flush=True)warn_budget_over_ram(engine_args(cfg) if "args" in cfg else []) # #1080+ note = card_note(cfg, env)+ if note:+ print(note, flush=True)if layer_split_of(cfg):try:split = layer_split_value(cfg) # #644: before the (minutes-long) startdiff --git a/serve/test_images.py b/serve/test_images.pyindex 2842e5230..95ad7fbe9 100644--- a/serve/test_images.py+++ b/serve/test_images.py@@ -96,6 +96,7 @@ class Command(unittest.TestCase):def test_env(self):env = images_api.image_env({"backend": "cuda", "gpu": 1, "env": {"X": 2}}, {"PATH": "p"})self.assertEqual((env["CUDA_VISIBLE_DEVICES"], env["CUDA_DEVICE_ORDER"], env["X"]), ("1", "PCI_BUS_ID", "2"))+ self.assertEqual(images_api.image_env({"backend": "cuda", "gpu": [1, 0]}, {})["CUDA_VISIBLE_DEVICES"], "1,0") # a list is a device listwith tempfile.TemporaryDirectory() as d:env = images_api.image_env({"backend": "hip", "gpu": 0, "lib_dirs": [d, d + "-missing"]}, {})self.assertEqual(env["HIP_VISIBLE_DEVICES"], "0")diff --git a/serve/test_server.py b/serve/test_server.pyindex 8bbe6e65b..4249585d4 100644--- a/serve/test_server.py+++ b/serve/test_server.py@@ -1726,6 +1726,19 @@ class GpuChoice(unittest.TestCase):self.assertEqual(plain.get("CUDA_VISIBLE_DEVICES"), os.environ.get("CUDA_VISIBLE_DEVICES"))self.assertEqual(plain.get("CUDA_DEVICE_ORDER"), os.environ.get("CUDA_DEVICE_ORDER"))+ def test_card_note_ties_the_engines_numbering_to_the_persons(self):+ """The engine counts the cards it is given from 0, so GPU 1 reads "GPU 0 / CUDA0" in its log: one line says which is which."""+ from serve.server import card_note, child_env+ one = {"gpu": 1, "args": []}+ self.assertIn("GPU 1", card_note(one, child_env(one)))+ self.assertIn("GPU 0 / CUDA0", card_note(one, child_env(one)))+ both = {"gpu": [1, 0], "args": [], "gpu_order": "as_given"}+ line = card_note(both, child_env(both))+ self.assertIn("GPU 1, then GPU 0", line)+ self.assertIn("CUDA0, CUDA1", line)+ for same in ({"gpu": 0, "args": []}, {"gpu": [0, 1], "args": [], "gpu_order": "as_given"}, {"args": []}, {"gpu": 1, "backend": "hip"}):+ self.assertIsNone(card_note(same, child_env(same)), same) # the numbers agree, or no card was chosen, or AMD (its own numbering)+def test_card_order(self):"""#1352: with an auto layer split the faster card (SMs x clock) goes last; ties, a manual split, ameasurement we lack, HIP and "gpu_order": "as_given" keep the config's order."""diff --git a/strata_cli/launch/app.py b/strata_cli/launch/app.pyindex 8a327b95a..29d575603 100644--- a/strata_cli/launch/app.py+++ b/strata_cli/launch/app.py@@ -174,7 +174,7 @@ class Ctx:from serve.server import engine_argsimport contextlibimport io- d = derive_config(apply_choices(cfg, s), s, bool_flags=tune_knobs.BOOL_FLAGS, gpus=cards_for_launch(s, self.catalog))+ d = derive_config(apply_choices(cfg, s), s, bool_flags=tune_knobs.BOOL_FLAGS, gpus=cards_for_launch(s, self.catalog, base=cfg))if s.parallel:d["parallel"] = s.parallelwith contextlib.redirect_stdout(io.StringIO()):diff --git a/strata_cli/launch/runner.py b/strata_cli/launch/runner.pyindex d8c223641..688459800 100644--- a/strata_cli/launch/runner.py+++ b/strata_cli/launch/runner.py@@ -214,17 +214,36 @@ def apply_cards(cfg: dict, gpus: list[int] | None) -> None:gpus = [int(g) for g in gpus]cfg["gpu"] = gpus[0] if len(gpus) == 1 else gpuscfg["gpus_asked"] = True+ cfg.pop("hip_ordinal", None) # an old AMD config's ordinal of ITS card would win over the card named nowif len(gpus) > 1:cfg["layer_split"] = cfg.get("layer_split") or "auto"else:cfg.pop("layer_split", None)-def cards_for_launch(s: S.LaunchSettings, catalog, ram_gb: float | None = None) -> list[int] | None:- """The cards this launch uses: the ones named (--gpu / --gpus, the Cards step), else the recommended ones by the shared fit rule- (strata_control/gpu_fit.py) when the computer has more than one card that counts, else None (one card: the installed config's own)."""+def installed_cards(base: dict | None) -> list[int] | None:+ """The cards the installed model's own config names after a choice (setup --gpu / --gpus, the app's card pick: "gpus_asked"), in its order;+ None when it names none or was never asked."""+ if not isinstance(base, dict) or not base.get("gpus_asked"):+ return None+ g = base.get("gpu")+ if g is None or g == "":+ return None+ try:+ cards = [int(str(x).strip()) for x in (g if isinstance(g, (list, tuple)) else str(g).split(",")) if str(x).strip() != ""]+ except ValueError:+ return None+ return cards or None+++def cards_for_launch(s: S.LaunchSettings, catalog, ram_gb: float | None = None, base: dict | None = None) -> list[int] | None:+ """The cards this launch uses: the ones named (--gpu / --gpus, the Cards step), else the cards the installed config was set to (a person's+ earlier choice, "gpu": 1 or [1, 0] - never replaced by the automatic pick, which would put the first of equal cards back), else the+ recommended ones by the shared fit rule (strata_control/gpu_fit.py) when the computer has more than one card that counts, else None."""if s.gpus:return [int(g) for g in s.gpus]+ if installed_cards(base):+ return Nonetry:from strata_control import gpu_fitcomp = catalog.computer()@@ -854,7 +873,7 @@ class Runner:from strata_control import tune_knobsbase = read_config(base_path)cfg = derive_config(base, self.s, bool_flags=tune_knobs.BOOL_FLAGS,- log=str(launch_dir() / f"{self.name}.engine.log"), gpus=cards_for_launch(self.s, self.catalog))+ log=str(launch_dir() / f"{self.name}.engine.log"), gpus=cards_for_launch(self.s, self.catalog, base=base))out = launch_dir() / f"{self.name}.json"write_private(out, cfg)self.config_path = outdiff --git a/strata_cli/launch/screens.py b/strata_cli/launch/screens.pyindex 537309373..917df146e 100644--- a/strata_cli/launch/screens.py+++ b/strata_cli/launch/screens.py@@ -544,6 +544,14 @@ class CardsScreen(Base):r = next((x for x in self.ctx.rows if (x.family, x.model) == (s.family, s.model)), None)return float(r.memory_gb) if r is not None and r.memory_gb else 44.0+ def installed(self) -> list[int] | None:+ """The cards the installed model was set to by a person's choice (kept unless this launch names others)."""+ from strata_cli.launch.runner import installed_cards+ try:+ return installed_cards(self.ctx.base_config())+ except Exception: # noqa: BLE001+ return None+def choices(self) -> dict:from strata_control import gpu_fitreturn gpu_fit.options(self.model_memory(), self.ctx.computer.ram_gb, self.ctx.computer.gpus, self.ctx.computer.engine_vendor)@@ -553,10 +561,11 @@ class CardsScreen(Base):items = []sel = 0cur = self.ctx.settings.gpus+ inst = self.installed()for i, o in enumerate(self.res["options"]):- label = o["label"] + (" (recommended)" if o["recommended"] else "")+ label = o["label"] + (" (recommended)" if o["recommended"] else "") + (" (as installed)" if inst and sorted(o["cards"]) == sorted(inst) else "")items.append(Item([label, Cell(o["words"], WARN if o["verdict"] in ("spills", "tight") else "")], key=o["cards"]))- if cur is None and o["recommended"] or (cur is not None and o["cards"] == list(cur)):+ if cur is None and (inst is None and o["recommended"] or inst is not None and sorted(o["cards"]) == sorted(inst)) or (cur is not None and o["cards"] == list(cur)):sel = iwide = max((len(str(i.cells[0])) for i in items), default=40)self.query_one(Menu).columns = [("", max(40, min(wide + 3, 70))), ("", 0)]@@ -587,7 +596,9 @@ class CardsScreen(Base):def commit(self) -> None:cur = self.menu().currentrec = next((o["cards"] for o in self.res["options"] if o["recommended"]), None)- self.ctx.settings.gpus = None if cur.key == rec else list(cur.key) # the recommended choice stays "let Strata choose" (the same rule in setup)+ inst = self.installed()+ keep = rec if inst is None else inst # "let Strata choose" = the recommendation, or the cards the installed model was set to+ self.ctx.settings.gpus = None if keep is not None and sorted(cur.key) == sorted(keep) else list(cur.key) # the recommended choice stays "let Strata choose" (the same rule in setup)# ------------------------------------------------------------------------------------------------ 3 folderdiff --git a/tests/terminal/test_launch.py b/tests/terminal/test_launch.pyindex 18fd91bd8..e471c9612 100644--- a/tests/terminal/test_launch.py+++ b/tests/terminal/test_launch.py@@ -580,6 +580,8 @@ class AdoptRules(unittest.TestCase):cfg = derive_config(split_base, S.LaunchSettings(), bool_flags=tune_knobs.BOOL_FLAGS)self.assertEqual(cfg["gpu"], [0, 1])self.assertEqual(base["gpu"], 0) # the installed config itself is never changed+ aged = derive_config({**base, "backend": "hip", "hip_ordinal": 1}, S.LaunchSettings(), bool_flags=tune_knobs.BOOL_FLAGS, gpus=[0])+ self.assertNotIn("hip_ordinal", aged) # a card named now is not overruled by an old config's ordinal# which cards: named ones win (one card too: --gpu 1 on a box that has them), else the fit rule on a box with several cardstwo = catalog(HW_2X16)self.assertEqual(cards_for_launch(S.LaunchSettings(family="qwen", model="IQ3_XXS", gpus=[1, 0]), two), [1, 0])@@ -596,6 +598,34 @@ class AdoptRules(unittest.TestCase):self.assertEqual(json.loads(out.read_text())["gpu"], 1)self.assertEqual(json.loads(installed.read_text())["gpu"], 0)+ def test_a_card_the_installed_model_was_set_to_is_not_replaced_by_the_automatic_pick(self):+ """Two equal cards: the fit rule recommends card 0, so a person's earlier choice (setup --gpu 1, the app's card pick: "gpus_asked")+ must survive a `strata launch` that names no cards - in its order, with "gpu_order": "as_given" too."""+ import json+ from tests.terminal.launch_support import HW_2X16+ from strata_cli.launch.runner import cards_for_launch, installed_cards+ with Home() as h:+ two = catalog(HW_2X16)+ plain = S.LaunchSettings(family="qwen", model="IQ3_XXS")+ chosen = {"exe": "component:strata-engine", "args": ["--max-context", "32768"], "gpu": 1, "gpus_asked": True, "port": 8080}+ self.assertEqual(installed_cards(chosen), [1])+ self.assertEqual(installed_cards({**chosen, "gpu": [1, 0], "gpu_order": "as_given"}), [1, 0])+ self.assertEqual(installed_cards({**chosen, "gpu": "1,0"}), [1, 0])+ self.assertIsNone(installed_cards({**chosen, "gpus_asked": False})) # never asked: setup's default, not a choice+ self.assertIsNone(installed_cards({"gpu": 1}))+ self.assertIsNone(cards_for_launch(plain, two, base=chosen)) # the installed choice stays+ self.assertEqual(cards_for_launch(plain, two, base={**chosen, "gpus_asked": False}), [0]) # no choice recorded: the fit rule+ self.assertEqual(cards_for_launch(S.LaunchSettings(family="qwen", model="IQ3_XXS", gpus=[0]), two, base=chosen), [0]) # --gpu 0 wins+ if True:+ installed = h.base / "strata-iq3_xxs.json"+ for gpu, extra in ((1, {}), ([1, 0], {"gpu_order": "as_given", "layer_split": "auto"})):+ base = {**chosen, "gpu": gpu, **extra}+ installed.write_text(json.dumps(base))+ r = Runner(S.LaunchSettings(family="qwen", model="IQ3_XXS", name="keep"), catalog=two)+ cfg = json.loads(r.build_config(installed).read_text())+ self.assertEqual(cfg["gpu"], gpu)+ self.assertEqual(cfg.get("gpu_order"), extra.get("gpu_order"))+def test_pictures_asked_but_still_off_is_said_in_plain_words(self):with Home() as h:cat = catalog(HW_12)