Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

launch: the card or cards chosen this time go into every launch's own config (--gpu, the Cards step, or the fit rule), so an installed model never keeps its old card and nothing is reinstalled

Niko12217bdc1ffb7248
diff --git a/strata_cli/launch/app.py b/strata_cli/launch/app.py
index 924cc20eb..6fe1c6c28 100644
--- a/strata_cli/launch/app.py
+++ b/strata_cli/launch/app.py
@@ -141,11 +141,11 @@ class Ctx:
return line + " (the engine command is shown once the model is installed)"
try:
from strata_control import tune_knobs
- from strata_cli.launch.runner import derive_config
+ from strata_cli.launch.runner import cards_for_launch, derive_config
from serve.server import engine_args
import contextlib
import io
- d = derive_config(cfg, s, bool_flags=tune_knobs.BOOL_FLAGS)
+ d = derive_config(cfg, s, bool_flags=tune_knobs.BOOL_FLAGS, gpus=cards_for_launch(s, self.catalog))
if s.parallel:
d["parallel"] = s.parallel
with contextlib.redirect_stdout(io.StringIO()):
diff --git a/strata_cli/launch/runner.py b/strata_cli/launch/runner.py
index b6fed20ec..a26a60c35 100644
--- a/strata_cli/launch/runner.py
+++ b/strata_cli/launch/runner.py
@@ -94,20 +94,9 @@ def read_config(path: Path) -> dict:
return json.loads(Path(path).read_text(encoding="utf-8-sig"))
-def gpus_of(desc_gpu, largest: int | None) -> list[int] | None:
- """The card numbers an installed config uses: a list, one number, or nothing written (setup then took the largest card)."""
- if isinstance(desc_gpu, list):
- return sorted(int(x) for x in desc_gpu)
- if isinstance(desc_gpu, int):
- return [desc_gpu]
- return [largest] if largest is not None else None
-
-
-def desc_matches(desc: dict, s: S.LaunchSettings, gpus: list[int] | None = None, largest: int | None = None) -> bool:
- """Does the installed model's config already carry the context, memory format, picture choice and graphics cards of these settings?
- `gpus`: the cards these settings mean (None: this computer has one card that counts, nothing to compare)."""
- if gpus is not None and gpus_of(desc.get("gpu"), largest) != sorted(gpus):
- return False
+def desc_matches(desc: dict, s: S.LaunchSettings) -> bool:
+ """Does the installed model's config already carry the context, memory format and picture choice of these settings?
+ (The graphics cards are not part of it: they go into each launch's own config, see apply_cards.)"""
if s.context and desc.get("context") != s.context:
return False
want_kv = s.kv or ("int8" if (s.context or 32768) > 8192 else "fp16")
@@ -137,8 +126,39 @@ def files_present(cfg: dict) -> list[str]:
return missing
-def derive_config(base: dict, s: S.LaunchSettings, *, bool_flags: frozenset[str], log: str | None = None) -> dict:
- """The private run config of this launch: the installed one plus the choices that are not setup's (host, port, key, advanced)."""
+def apply_cards(cfg: dict, gpus: list[int] | None) -> None:
+ """The graphics cards of THIS launch go into its run config, whatever card the model was installed with (no reinstall): one card is a number,
+ several are a list with the automatic layer split. None: the installed config's own choice stays."""
+ if not gpus:
+ return
+ gpus = [int(g) for g in gpus]
+ cfg["gpu"] = gpus[0] if len(gpus) == 1 else gpus
+ cfg["gpus_asked"] = True
+ if len(gpus) > 1:
+ cfg["layer_split"] = cfg.get("layer_split") or "auto"
+ else:
+ cfg.pop("layer_split", None)
+
+
+def cards_for_launch(s: S.LaunchSettings, catalog, ram_gb: float | None = None) -> list[int] | None:
+ """The cards this launch uses: the ones named (--gpu / --gpus, the Cards step), else the recommended ones by the shared fit rule
+ (strata_control/gpu_fit.py) when the computer has more than one card that counts, else None (one card: the installed config's own)."""
+ if s.gpus:
+ return [int(g) for g in s.gpus]
+ try:
+ from strata_control import gpu_fit
+ comp = catalog.computer()
+ if len(comp.usable_gpus) < 2:
+ return None
+ row = next((r for r in catalog.rows() if (r.family, r.model) == (s.family, s.model)), None)
+ mem = float(row.memory_gb) if row is not None and row.memory_gb else 44.0
+ return gpu_fit.recommended_cards(mem, comp.ram_gb, comp.gpus) or None
+ except Exception: # noqa: BLE001 - never block a launch over the choice of cards
+ return None
+
+
+def derive_config(base: dict, s: S.LaunchSettings, *, bool_flags: frozenset[str], log: str | None = None, gpus: list[int] | None = None) -> dict:
+ """The private run config of this launch: the installed one plus the choices that are not setup's (host, port, key, cards, advanced)."""
from strata_control import tune_knobs
cfg = json.loads(json.dumps(base))
cfg["host"] = acc.host_for(s.access)
@@ -150,6 +170,7 @@ def derive_config(base: dict, s: S.LaunchSettings, *, bool_flags: frozenset[str]
cfg.setdefault("exe", "component:strata-engine")
if log:
cfg["log"] = log
+ apply_cards(cfg, gpus)
if s.extra:
a_changes, e_changes = S.as_argv_changes(s.extra, bool_flags)
cfg["args"] = tune_knobs.apply_to_argv([str(x) for x in cfg.get("args") or []], a_changes)
@@ -382,24 +403,6 @@ class Runner:
desc = self.catalog.strata.describe_config(path)
return path, desc
- def wanted_gpus(self) -> tuple[list[int] | None, int | None]:
- """(the cards these settings mean, the largest card): the cards named, else the recommended ones by the shared fit rule
- (strata_control/gpu_fit.py); (None, None) when this computer has one card that counts."""
- try:
- from strata_control import gpu_fit
- comp = self.catalog.computer()
- keep = comp.usable_gpus
- if len(keep) < 2:
- return None, None
- largest = keep[0].get("index")
- if self.s.gpus:
- return sorted(int(g) for g in self.s.gpus), largest
- row = next((r for r in self.catalog.rows() if (r.family, r.model) == (self.s.family, self.s.model)), None)
- mem = float(row.memory_gb) if row is not None and row.memory_gb else 44.0
- return sorted(gpu_fit.recommended_cards(mem, comp.ram_gb, comp.gpus)), largest
- except Exception: # noqa: BLE001 - never block a launch over this comparison
- return None, None
-
def download_gb(self) -> float:
"""How much would be downloaded for this launch right now: 0 when the model is installed with these choices (or is the person's own files)."""
s = self.s
@@ -411,7 +414,7 @@ class Runner:
if lost and all("mmproj" in Path(x).name.lower() for x in lost):
return 0.9 # only the picture reader's file is gone
if path is not None and desc is not None and not lost:
- if desc_matches(desc, s, *self.wanted_gpus()):
+ if desc_matches(desc, s):
return 0.0
# the model is on disk: what differs is the context / memory format (nothing to download) or the pictures (the 0.9 GB reader file)
return 0.9 if (s.vision != "off" and (desc.get("images") or "off") == "off") else 0.0
@@ -427,7 +430,7 @@ class Runner:
if not (s.family and s.model):
raise LaunchError("No model was chosen.", "Choose a model, or use --model NAME (strata launch --help lists them).", 2)
path, desc = self.installed_state()
- if path is not None and desc is not None and desc_matches(desc, s, *self.wanted_gpus()) and not files_present(read_config(path)):
+ if path is not None and desc is not None and desc_matches(desc, s) and not files_present(read_config(path)):
self.emit("model", f"{s.family} {s.model} is already installed with these choices.", 100)
return path
title = f"{s.family} {s.model}"
@@ -544,7 +547,7 @@ class Runner:
from strata_control import tune_knobs
base = read_config(base_path)
cfg = derive_config(base, self.s, bool_flags=tune_knobs.BOOL_FLAGS,
- log=str(launch_dir() / f"{self.name}.engine.log"))
+ log=str(launch_dir() / f"{self.name}.engine.log"), gpus=cards_for_launch(self.s, self.catalog))
out = launch_dir() / f"{self.name}.json"
write_private(out, cfg)
self.config_path = out
diff --git a/tests/terminal/test_launch.py b/tests/terminal/test_launch.py
index 9ada36d86..76b7d2382 100644
--- a/tests/terminal/test_launch.py
+++ b/tests/terminal/test_launch.py
@@ -455,23 +455,40 @@ class AdoptRules(unittest.TestCase):
with mock.patch.object(r, "installed_state", return_value=(cfgfile, {"context": 32768, "kv": "int8", "images": "cpu"})):
self.assertAlmostEqual(r.download_gb(), 0.9)
- def test_the_installed_config_must_carry_the_chosen_cards(self):
+ def test_the_cards_of_this_launch_go_into_its_config_without_reinstalling(self):
from tests.terminal.launch_support import HW_2X16
- with Home():
- cat = catalog(HW_2X16)
- base = {"context": 32768, "kv": "int8", "images": "off"}
- auto = Runner(S.LaunchSettings(family="qwen", model="IQ3_XXS", context=32768), catalog=cat)
- self.assertEqual(auto.wanted_gpus(), ([0], 0)) # the recommended: the largest card + RAM
- both = Runner(S.LaunchSettings(family="qwen", model="IQ3_XXS", context=32768, gpus=[0, 1]), catalog=cat)
- self.assertEqual(both.wanted_gpus(), ([0, 1], 0))
- # a config written for one card (no "gpu" key: setup took the largest) is right for the default and wrong for the split
- self.assertTrue(desc_matches({**base, "gpu": None}, auto.s, *auto.wanted_gpus()))
- self.assertFalse(desc_matches({**base, "gpu": None}, both.s, *both.wanted_gpus()))
- self.assertTrue(desc_matches({**base, "gpu": [1, 0]}, both.s, *both.wanted_gpus()))
- self.assertFalse(desc_matches({**base, "gpu": [0, 1]}, auto.s, *auto.wanted_gpus()))
- with Home():
- one = Runner(S.LaunchSettings(family="qwen", model="IQ3_XXS", context=32768), catalog=catalog(HW_12))
- self.assertEqual(one.wanted_gpus(), (None, None)) # one card: nothing to compare
+ from strata_cli.launch.runner import apply_cards, cards_for_launch
+ from strata_control import tune_knobs
+ base = {"exe": "component:strata-engine", "args": ["--max-context", "32768"], "gpu": 0, "port": 8080}
+ with Home() as h:
+ # an installed single-card config, then the split chosen now
+ cfg = derive_config(base, S.LaunchSettings(), bool_flags=tune_knobs.BOOL_FLAGS, gpus=[0, 1])
+ self.assertEqual((cfg["gpu"], cfg["layer_split"]), ([0, 1], "auto"))
+ self.assertTrue(cfg["gpus_asked"])
+ # an installed split config, then one named card now
+ split_base = {**base, "gpu": [0, 1], "layer_split": "auto"}
+ cfg = derive_config(split_base, S.LaunchSettings(), bool_flags=tune_knobs.BOOL_FLAGS, gpus=[1])
+ self.assertEqual(cfg["gpu"], 1)
+ self.assertNotIn("layer_split", cfg)
+ # nothing chosen and nothing to recommend: the installed choice stays
+ cfg = derive_config(split_base, S.LaunchSettings(), bool_flags=tune_knobs.BOOL_FLAGS)
+ self.assertEqual(cfg["gpu"], [0, 1])
+ self.assertEqual(base["gpu"], 0) # the installed config itself is never changed
+ # which cards: named ones win (one card too: --gpu 1 on a box that has them), else the fit rule on a box with several cards
+ two = catalog(HW_2X16)
+ self.assertEqual(cards_for_launch(S.LaunchSettings(family="qwen", model="IQ3_XXS", gpus=[1, 0]), two), [1, 0])
+ self.assertEqual(cards_for_launch(S.LaunchSettings(family="qwen", model="IQ3_XXS"), two), [0]) # fits one card + RAM
+ one = catalog(HW_12)
+ self.assertEqual(cards_for_launch(S.LaunchSettings(family="qwen", model="IQ3_XXS", gpus=[1]), one), [1])
+ self.assertIsNone(cards_for_launch(S.LaunchSettings(family="qwen", model="IQ3_XXS"), one))
+ # the launch's own config is written with them (Runner.build_config), the installed file is untouched
+ import json
+ installed = h.base / "strata-iq3_xxs.json"
+ installed.write_text(json.dumps(base))
+ r = Runner(S.LaunchSettings(family="qwen", model="IQ3_XXS", name="cards", gpus=[1]), catalog=two)
+ out = r.build_config(installed)
+ self.assertEqual(json.loads(out.read_text())["gpu"], 1)
+ self.assertEqual(json.loads(installed.read_text())["gpu"], 0)
def test_pictures_asked_but_still_off_is_said_in_plain_words(self):
with Home() as h: