Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Terminal: pictures are on by default when the model and the engine can read them (card for CUDA and HIP, CPU otherwise); --vision off turns them off

Niko1221d060783fbb93
diff --git a/strata_cli/launch/catalog.py b/strata_cli/launch/catalog.py
index bf80ee80f..18eec408c 100644
--- a/strata_cli/launch/catalog.py
+++ b/strata_cli/launch/catalog.py
@@ -358,6 +358,44 @@ class Catalog:
pass
return rec
+ # ---- pictures
+ def engine_backend(self) -> str | None:
+ """The installed engine's backend (cuda, hip, sycl, cpu, metal), else the one this computer's graphics card needs, else None."""
+ try:
+ from strata_control import engine_component as ec
+ rep = ec.status()
+ if rep.get("usable"):
+ return (rep.get("engine") or {}).get("backend")
+ except Exception: # noqa: BLE001
+ pass
+ for g in self.computer().usable_gpus:
+ v = str(g.get("vendor") or "").lower()
+ if g.get("unified_gb") or v == "apple":
+ return "metal"
+ return {"nvidia": "cuda", "amd": "hip", "intel": "sycl"}.get(v)
+ return None
+
+ def reader_state(self) -> str:
+ """"present" when the installed engine carries the picture reader (strata-vision), "missing" when an engine is installed without it,
+ "unknown" before any engine is installed (the first launch fetches one that has it)."""
+ try:
+ from strata_control import engine_component as ec
+ rep = ec.status()
+ exe = (rep.get("engine") or {}).get("exe") if rep.get("usable") else None
+ except Exception: # noqa: BLE001
+ exe = None
+ if not exe:
+ return "unknown"
+ folder = Path(str(exe)).parent
+ return "present" if any((folder / n).is_file() for n in ("strata-vision", "strata-vision.exe")) else "missing"
+
+ def picture_default(self, images: bool) -> str:
+ """The picture setting nobody chose: off for a model that cannot read pictures or an engine without the reader; else on, with the
+ reader on the graphics card for the CUDA and HIP builds and on the CPU for the others."""
+ if not images or self.reader_state() == "missing":
+ return "off"
+ return "gpu" if self.engine_backend() in ("cuda", "hip") else "cpu"
+
# ---- installed models
def installed_configs(self) -> list[dict]:
out, seen = [], set()
diff --git a/strata_cli/launch/cmd.py b/strata_cli/launch/cmd.py
index 29c6dd1d6..1c00e5b00 100644
--- a/strata_cli/launch/cmd.py
+++ b/strata_cli/launch/cmd.py
@@ -85,7 +85,8 @@ def add_parsers(sp: argparse._SubParsersAction) -> list[argparse.ArgumentParser]
g.add_argument("--context", type=_context, metavar="SIZE", help="how much text it remembers at once: 32k, 64k, 128k, 256k or a number of tokens")
g.add_argument("--kv", type=_choice("a memory format", S.KV_MODES), metavar="FORMAT", help="conversation memory format: int8 (default), q4_0 (half the memory), k8v4, fp16")
g.add_argument("--vision", type=_choice("a picture setting", S.VISION_MODES), metavar="off|on|gpu|cpu",
- help="let the model read pictures (default off)")
+ help="let the model read pictures. Default: on when the model can read pictures and the engine has the picture reader "
+ "(on the graphics card for NVIDIA and AMD, on the CPU otherwise); off turns it off")
g.add_argument("--parallel", type=_parallel, metavar="N", help="requests answered at the same time, 1 to 8 (default: fit to your graphics card)")
g.add_argument("--gpu", metavar="LIST", help="graphics card numbers to use, e.g. 0 or 0,1 (default: the engine's choice)")
g.add_argument("--tune", type=_choice("a tuning mode", S.TUNE_MODES), metavar="quick|thorough|skip", help="find the fastest settings first (default: skip)")
@@ -238,9 +239,14 @@ def apply_defaults(s: S.LaunchSettings, catalog) -> tuple[S.LaunchSettings, list
notices.append(S.Notice("warn", f"{r.title} does not fit this computer.", "It can still start; expect it to be slow or to run out of memory."))
elif r.verdict == "tight":
notices.append(S.Notice("warn", f"{r.title} is a tight fit: {r.words}.", "Close other programs before you start."))
- if s.vision != "off" and not r.images:
+ if s.vision not in (None, "off") and not r.images:
notices.append(S.Notice("warn", f"{r.title} cannot read pictures yet, so pictures are off.", ""))
s.vision = "off"
+ if s.vision is None: # nobody chose: on when the model and the engine can, silently off otherwise
+ s.vision = catalog.picture_default(r.images)
+ if s.vision == "off" and r.images and catalog.reader_state() == "missing":
+ notices.append(S.Notice("warn", "The engine installed here has no picture reader, so pictures are off.",
+ "Update the engine to read pictures, or add --vision off to silence this."))
notices += S.check_context(s, max_fit_tokens=None)
if s.access != "local" and not s.api_key:
from strata_cli.launch import access
diff --git a/strata_cli/launch/runner.py b/strata_cli/launch/runner.py
index 0394cc9ab..cc8591c16 100644
--- a/strata_cli/launch/runner.py
+++ b/strata_cli/launch/runner.py
@@ -125,6 +125,8 @@ def desc_matches(desc: dict, s: S.LaunchSettings) -> bool:
if desc.get("kv") not in (None, want_kv) and (s.context or 32768) > 8192:
return False
images = desc.get("images") or "off"
+ if s.vision is None:
+ return True # no preference yet: the run settles it before it compares
if (s.vision == "off") != (images == "off"):
return False
if s.vision in ("gpu", "cpu") and images != s.vision:
@@ -403,10 +405,19 @@ class Runner:
return note
# ---- 2. the model
+ def resolved_vision(self) -> str:
+ """The picture setting of this launch: the chosen one, or (nothing chosen) the default for this model and engine (catalog.picture_default)."""
+ s = self.s
+ if s.vision is None:
+ row = next((r for r in self.catalog.rows() if (r.family, r.model) == (s.family, s.model)), None)
+ s.vision = self.catalog.picture_default(bool(row.images) if row is not None else False)
+ return s.vision
+
def vision_word(self) -> str:
"""What setup is told for the picture choice. "on" is setup's "yes": setup (Control mode) picks the CPU encoder on a card under 24 GB
and on AMD, the GPU encoder on a bigger NVIDIA card, and downloads the pinned picture-reader file (mmproj) with the model."""
s = self.s
+ self.resolved_vision()
if s.vision == "off":
return "no"
return "yes" if s.vision == "on" else s.vision
@@ -496,6 +507,7 @@ class Runner:
"""What this launch still needs for the model, in numbers and words (strata_cli/launch/modelstate.py)."""
from strata_cli.launch import modelstate
s = self.s
+ self.resolved_vision()
path, desc = (None, None)
if s.family and s.model and not s.model_path:
path, desc = self.installed_state()
@@ -521,6 +533,7 @@ class Runner:
s, cat = self.s, self.catalog
if s.model_path and not (s.family and s.model):
self.resolve_own_files()
+ self.resolved_vision()
if not (s.family and s.model):
raise LaunchError("No model was chosen.", "Choose a model, or use --model NAME (strata launch --help lists them).", 2)
path, desc = self.installed_state()
diff --git a/strata_cli/launch/screens.py b/strata_cli/launch/screens.py
index 1391b1c51..119c5fb3c 100644
--- a/strata_cli/launch/screens.py
+++ b/strata_cli/launch/screens.py
@@ -808,11 +808,7 @@ class PicturesScreen(Base):
def reader(self) -> str:
""""present" when the installed engine carries the picture reader (strata-vision), "missing" when an engine is installed without it,
"unknown" before any engine is installed (the first launch fetches it)."""
- exe = self.ctx.engine_exe()
- if not exe:
- return "unknown"
- folder = Path(str(exe)).parent
- return "present" if any((folder / n).is_file() for n in ("strata-vision", "strata-vision.exe")) else "missing"
+ return self.ctx.catalog.reader_state()
def on_mount(self) -> None:
s = self.ctx.settings
@@ -823,18 +819,19 @@ class PicturesScreen(Base):
items.append(Item(["Yes, read pictures" + (" (this engine has no picture reader)" if self.reader() == "missing" else "")], key="on"))
else:
items.append(Item([Cell("Yes, read pictures: not available for this model yet", MUTE)], key="on", enabled=False))
- sel = 1 if s.vision != "off" and self.supported else 0
+ sel = 1 if s.vision != "off" and self.supported and self.reader() != "missing" else 0 # pictures are on unless someone said off
self.query_one(Menu).set_items(items, sel)
self.paint()
def explain(self) -> Text:
- vram = self.ctx.computer.main_vram_gb
- extra = ("a 0.9 GB download. On a card under 24 GB the picture reader runs on the CPU (a few seconds per picture, nothing "
- "taken from the graphics card). On a bigger card it runs on the GPU and keeps about 1.4 GB free, costing a few % of text speed."
- if vram < 24 else "a 0.9 GB download, and about 1.4 GB of graphics memory kept free for the picture reader (text is a few % slower).")
+ on_card = self.ctx.catalog.picture_default(True) == "gpu"
+ extra = ("a 0.9 GB download (if it is not on this computer yet), and about 1.4 GB of graphics memory kept free for the picture reader "
+ "(text is a few % slower)." if on_card else
+ "a 0.9 GB download (if it is not on this computer yet). The picture reader runs on the CPU: a few seconds per picture, nothing "
+ "taken from the graphics card.")
if not self.supported:
return Text("This model cannot read pictures yet, so pictures stay off. Pick another model if you need them.", style=WARN)
- out = Text("Pictures add " + extra + " Say No if you only chat in text.", style=MUTE)
+ out = Text("Pictures are on by default. They add " + extra + " Say No if you only chat in text.", style=MUTE)
if self.reader() == "missing":
out.append("\nHeads up: the engine installed here does not carry the picture reader, so the model would answer text only. "
"Update the engine, or say No.", style=WARN)
@@ -842,7 +839,11 @@ class PicturesScreen(Base):
return out
def commit(self) -> None:
- self.ctx.settings.vision = self.menu().current.key
+ key = self.menu().current.key
+ if key == "on": # yes: the reader on the card (NVIDIA, AMD) or the CPU, as the default says
+ device = self.ctx.catalog.picture_default(True)
+ key = device if device != "off" else "on"
+ self.ctx.settings.vision = key
# ------------------------------------------------------------------------------------------------ 6 parallel
diff --git a/strata_cli/launch/settings.py b/strata_cli/launch/settings.py
index 87f33e046..ca4779ab8 100644
--- a/strata_cli/launch/settings.py
+++ b/strata_cli/launch/settings.py
@@ -83,7 +83,7 @@ class LaunchSettings:
gpus: list[int] | None = None # None = the engine's choice (all usable cards)
context: int | None = None
kv: str | None = None # None = int8 above 8K, as setup
- vision: str = "off" # off | on | gpu | cpu
+ vision: str | None = None # None = the default (on when the model can read pictures and the engine has the reader), off | on | gpu | cpu
parallel: int | None = None # None = fit to the card
tune: str = "skip" # quick | thorough | skip
tune_allow_small: bool = False # the "answer-changing" tick, off by default, as in the app
@@ -259,7 +259,8 @@ def flags_text(s: LaunchSettings, *, preset: str | None = None) -> str:
parts.append(f"--context {context_label(s.context).lower()}")
if s.kv:
parts.append(f"--kv {s.kv}")
- parts.append(f"--vision {s.vision}")
+ if s.vision:
+ parts.append(f"--vision {s.vision}")
if s.parallel:
parts.append(f"--parallel {s.parallel}")
parts.append(f"--tune {s.tune}")
diff --git a/tests/terminal/test_launch.py b/tests/terminal/test_launch.py
index 819f8548c..5121be785 100644
--- a/tests/terminal/test_launch.py
+++ b/tests/terminal/test_launch.py
@@ -291,9 +291,48 @@ class SetupArgs(unittest.TestCase):
def runner(self, **kw):
with Home():
cat = catalog(HW_12)
+ kw.setdefault("vision", "off")
s = S.LaunchSettings(family="qwen", model="IQ3_XXS", context=32768, kv="q4_0", **kw)
return Runner(s, catalog=cat).setup_args()
+ def test_pictures_default_on_the_card_for_nvidia_and_amd_and_on_the_cpu_for_the_rest(self):
+ from tests.terminal.launch_support import HW_12 as NV
+ amd = {**NV, "gpus": [{"index": 0, "name": "AMD Radeon RX 9070 XT", "vram_gb": 16.0, "usable": True, "vendor": "amd"}]}
+ intel = {**NV, "gpus": [{"index": 0, "name": "Intel Arc B580", "vram_gb": 12.0, "usable": True, "vendor": "intel"}]}
+ with Home():
+ for hw, want in ((NV, "gpu"), (amd, "gpu"), (intel, "cpu")):
+ cat = catalog(hw)
+ r = Runner(S.LaunchSettings(family="qwen", model="IQ3_XXS", context=32768), catalog=cat)
+ self.assertEqual(r.setup_args()[r.setup_args().index("--vision") + 1], want, hw["gpus"][0]["vendor"])
+ self.assertEqual(r.s.vision, want)
+ off = Runner(S.LaunchSettings(family="qwen", model="IQ3_XXS", context=32768, vision="off"), catalog=catalog(NV))
+ self.assertEqual(off.setup_args()[off.setup_args().index("--vision") + 1], "no")
+
+ def test_a_model_that_cannot_read_pictures_stays_off_without_a_question(self):
+ import dataclasses
+ with Home():
+ cat = catalog(HW_12)
+ real = cat.rows()
+ rows = [dataclasses.replace(r, images=False) for r in real]
+ with mock.patch.object(type(cat), "rows", lambda self, *a, **k: rows):
+ r = Runner(S.LaunchSettings(family="qwen", model="IQ3_XXS", context=32768), catalog=cat)
+ self.assertEqual(r.resolved_vision(), "off")
+
+ def test_an_engine_without_the_reader_defaults_to_off(self):
+ with Home():
+ cat = catalog(HW_12)
+ with mock.patch.object(type(cat), "reader_state", lambda self: "missing"):
+ self.assertEqual(cat.picture_default(True), "off")
+
+ def test_the_command_defaults_pictures_on_and_off_still_turns_them_off(self):
+ from strata_cli.launch.cmd import apply_defaults
+ with Home():
+ cat = catalog(HW_12)
+ s, _ = apply_defaults(S.LaunchSettings(family="qwen", model="IQ3_XXS"), cat)
+ self.assertEqual(s.vision, "gpu")
+ s, _ = apply_defaults(S.LaunchSettings(family="qwen", model="IQ3_XXS", vision="off"), cat)
+ self.assertEqual(s.vision, "off")
+
def test_the_apps_own_installer_arguments(self):
import tempfile
own = tempfile.mkdtemp()
diff --git a/tests/terminal/test_launch_tui.py b/tests/terminal/test_launch_tui.py
index d523c1f1d..a9aba6f6a 100644
--- a/tests/terminal/test_launch_tui.py
+++ b/tests/terminal/test_launch_tui.py
@@ -422,13 +422,22 @@ class PicturesParallelTuning(Base):
await press(pilot, *self.reach_pictures())
t = text_of(app)
self.assertIn("No pictures", t)
+ self.assertIn("Pictures are on by default", flat(app))
self.assertIn("0.9 GB download", t)
- self.assertIn("runs on the CPU", t) # a 12 GB card
- await press(pilot, "down", "enter")
- self.assertEqual(ctx.settings.vision, "on")
+ self.assertIn("graphics memory kept free", flat(app)) # the NVIDIA reader runs on the card
+ self.assertEqual(app.screen.menu().index, 1) # "Yes" is preselected
+ await press(pilot, "enter")
+ self.assertEqual(ctx.settings.vision, "gpu")
self.assertIn("How many requests at the same time?", text_of(app))
self.run_app(case)
+ def test_pictures_no_is_one_key_away(self):
+ async def case(app, pilot, ctx):
+ await press(pilot, *self.reach_pictures())
+ await press(pilot, "up", "enter")
+ self.assertEqual(ctx.settings.vision, "off")
+ self.run_app(case)
+
def test_pictures_not_offered_for_a_model_without_them(self):
store.save(S.LaunchSettings(name="G", family="gyro", model="Gyro-S", context=32768))