Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Mac pictures: strata-vision with Metal built in, Mac setup and launch turn pictures on

Niko1221cf4749276dab
diff --git a/metal/setup_mac.py b/metal/setup_mac.py
index 1d4717ade..4d2554466 100644
--- a/metal/setup_mac.py
+++ b/metal/setup_mac.py
@@ -176,7 +176,7 @@ def build_vision(llama: Path) -> Path:
with open(log, "w", encoding="utf-8") as f:
for cmd in ([tool("cmake"), "-S", str(ROOT / "tools" / "vision"), "-B", str(ROOT / "build-vision"), "-G", "Ninja",
"-DCMAKE_BUILD_TYPE=Release", f"-DCMAKE_MAKE_PROGRAM={tool('ninja')}", f"-DLLAMA_DIR={llama}",
- "-DSTRATA_VISION_CUDA=OFF", "-DSTRATA_PORTABLE=OFF"],
+ "-DSTRATA_VISION_CUDA=OFF", "-DSTRATA_VISION_METAL=ON", "-DSTRATA_PORTABLE=OFF"],
[tool("ninja"), "-C", str(ROOT / "build-vision"), "strata-vision"]):
if subprocess.call(cmd, stdout=f, stderr=subprocess.STDOUT, env=env) != 0:
S.fail(f"the image encoder did not build (see {log})")
@@ -343,8 +343,9 @@ def main(argv=None) -> int:
ap.add_argument("--check", action="store_true")
ap.add_argument("--update", action="store_true")
ap.add_argument("--build", action="store_true", help="build the engine again even when it is current")
- ap.add_argument("--vision", choices=["yes", "no"],
- help="let the model read images (the image encoder on the GPU, built here; default: asked, no)")
+ ap.add_argument("--vision", choices=["yes", "no", "gpu", "cpu"],
+ help="let the model read images (the image encoder, built here; yes = gpu: it reads on the Mac's GPU, cpu is ~100x "
+ "slower; default: asked, yes)")
a, rest = ap.parse_known_args(argv)
if rest:
S.warn("not used on a Mac: " + " ".join(rest))
@@ -394,7 +395,12 @@ def main(argv=None) -> int:
for n in model_note(model, info["ram_gb"]):
S.warn(n)
vision = a.vision or ("yes" if S.ask("Should the model also read images? (the image encoder, ~1 GB more)",
- ["y", "n"], "n", a.yes) == "y" else "no")
+ ["y", "n"], "y", a.yes) == "y" else "no") # on by default (strata_control/vision_reader.py)
+ if vision == "cpu":
+ from strata_control import vision_reader as VR
+ S.warn(VR.MAC_CPU_NOTE)
+ vision_gpu = vision != "cpu"
+ vision = "no" if vision == "no" else "yes"
ctx = a.context or recommend_context(info["ram_gb"])
if ctx > TESTED_CONTEXT:
S.say(f" context {ctx}: measured on the Mac up to {TESTED_CONTEXT} (needles found at 8K-32K); longer is kept")
@@ -519,7 +525,7 @@ def main(argv=None) -> int:
cfg["env"] = {"STRATA_METAL_STREAM": "1" if stream else "0"} # the engine follows setup's measured choice
if vision == "yes":
cfg["args"] += ["--vision"]
- cfg["vision"] = {"exe": str(vexe), "mmproj": str(mmproj), "model": str(files[0]), "gpu": True,
+ cfg["vision"] = {"exe": str(vexe), "mmproj": str(mmproj), "model": str(files[0]), "gpu": vision_gpu,
"max_tokens": 1024}
if a.host:
cfg["host"] = a.host
diff --git a/packaging/build_vision_mac.sh b/packaging/build_vision_mac.sh
new file mode 100644
index 000000000..f3544df13
--- /dev/null
+++ b/packaging/build_vision_mac.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+# packaging/build_vision_mac.sh - the picture reader for the Mac engine package: strata-vision for macOS arm64, with ggml's Metal
+# backend built in (`--gpu` reads pictures on the Mac's GPU, without it on the CPU). One program for both: Apple Silicon always has
+# the Metal framework, the Metal engine needs it too. It is built against the llama.cpp commit the engine is pinned to.
+#
+# packaging/build_vision_mac.sh --llama <llama.cpp checkout at the pin> --out <folder> [--build <build folder>] [--jobs N]
+#
+# <folder>/strata-vision is the program; copy it beside `strata` in the engine package's --src before packaging/package_engine.py
+# (`strata-vision` is one of the files it takes) and sign it with the rest (packaging/mac/sign-and-notarize.sh signs every Mach-O).
+# Needs cmake and ninja on PATH and the Command Line Tools. The build is portable (no -march: it runs on an M1 as on an M5).
+set -euo pipefail
+here="$(cd "$(dirname "$0")" && pwd)"
+llama="" out="" build="" jobs=""
+while [ $# -gt 0 ]; do
+ case "$1" in
+ --llama) llama="$2"; shift 2 ;;
+ --out) out="$2"; shift 2 ;;
+ --build) build="$2"; shift 2 ;;
+ --jobs) jobs="$2"; shift 2 ;;
+ *) echo "unknown argument $1" >&2; exit 2 ;;
+ esac
+done
+[ -n "$llama" ] && [ -n "$out" ] || { echo "usage: $0 --llama <llama.cpp> --out <folder> [--build <folder>] [--jobs N]" >&2; exit 2; }
+[ "$(uname -s)" = Darwin ] && [ "$(uname -m)" = arm64 ] || { echo "this builds on an Apple Silicon Mac" >&2; exit 1; }
+build="${build:-$out/build-vision}"
+mkdir -p "$out"
+cmake -S "$here/../tools/vision" -B "$build" -G Ninja -DCMAKE_BUILD_TYPE=Release -DLLAMA_DIR="$llama" \
+ -DSTRATA_PORTABLE=ON -DSTRATA_VISION_METAL=ON -DSTRATA_VISION_CUDA=OFF
+ninja -C "$build" ${jobs:+-j "$jobs"} strata-vision
+cp "$build/bin/strata-vision" "$out/strata-vision"
+echo "built: $out/strata-vision"
+# what it links: only Apple's own libraries (binary_deps.py in the package step checks the same)
+otool -L "$out/strata-vision" | tail -n +2 | awk '{print " " $1}'
+if otool -L "$out/strata-vision" | tail -n +2 | awk '{print $1}' | grep -vE '^(/usr/lib/|/System/Library/)' ; then
+ echo "refusing: the reader links a library that is not part of macOS" >&2
+ exit 1
+fi
+"$out/strata-vision" --version
diff --git a/setup.py b/setup.py
index c7a2d32aa..d501ba303 100644
--- a/setup.py
+++ b/setup.py
@@ -6945,6 +6945,35 @@ def preview_family(a) -> int:
return 0
+def mac_vision(asked, model: str, ram: float) -> str:
+ """The picture setting of a Mac's setup: "gpu", "cpu" or "none". Pictures are on wherever the model can read them (owner rule
+ 2026-10-18) and the reader follows strata_control/vision_reader.py: on a Mac, the GPU (Metal) reader, because the memory is shared
+ (no card to keep free) and the CPU reader is ~100x slower. --vision cpu is kept, with a note (recommend, never force)."""
+ from strata_control import vision_reader as VR
+ if asked in ("no", "none", "off"):
+ return "none"
+ if asked is None and not images_default(model, "qwen", ram):
+ return "none"
+ if MODELS.get(model, {}).get("vision") is False:
+ if asked is not None:
+ warn(f"images are not available with {model} yet: off")
+ return "none"
+ mode = VR.wanted(asked, None, unified=True)
+ if mode == "cpu":
+ warn(VR.MAC_CPU_NOTE)
+ return mode
+
+
+def mac_reader(eng: Path, vision: str):
+ """The picture reader the Mac engine folder carries (strata-vision: Metal built in), for `vision` "gpu" or "cpu". A folder
+ without it gets pictures off, said once."""
+ from strata_control import vision_reader as VR
+ r = VR.choose(eng, vision, windows=False, mac=True)
+ if r.mode == "none":
+ warn(r.note or "this engine has no picture reader: pictures off")
+ return r
+
+
def mac_control_setup(a) -> int:
"""The app (and the Terminal's `strata launch`) on a Mac, STRATA_CONTROL_INSTALL=1: the same job as on Windows and Linux. It
downloads and prepares the model files and writes a run config whose engine is "component:strata-engine"; it builds nothing
@@ -6983,8 +7012,7 @@ def mac_control_setup(a) -> int:
fail(f"{model} is not offered on a Mac in the app", "choose one of: " + ", ".join(app_sizes))
for n in MF.model_note(model, ram):
warn(n)
- if a.vision in ("yes", "gpu", "cpu"):
- warn("the Mac engine has no picture reader yet: images off for this model (text works as usual)")
+ vision = mac_vision(a.vision, model, ram)
ctx = a.context or MF.recommend_context(ram)
if ctx > MF.TESTED_CONTEXT:
say(f" context {ctx}: measured on the Mac up to {MF.TESTED_CONTEXT} (needles found at 8K-32K); longer is kept")
@@ -6997,13 +7025,21 @@ def mac_control_setup(a) -> int:
from strata_control import engine_component as EC
llama = get_llama_cpp() # gguf-py for the preparation tools only: no git, no compiler
ok(f"llama.cpp {LLAMA_CPP_COMMIT[:7]} (gguf-py for the preparation tools)")
+ reader = None
try:
found = EC.resolve(full=False)
ok(f"engine component: {found.version} ({found.platform}-{found.backend})")
+ if vision != "none":
+ reader = mac_reader(Path(found.root), vision)
+ vision = reader.mode
except EC.EngineMissing as e:
warn(f"no Metal engine is installed for this Mac yet ({e}). The model is downloaded and prepared now; the engine comes "
"with the app or its update feed, and is looked up when the model loads")
+ if vision != "none":
+ warn("pictures are off for now: the picture reader comes with the engine (run setup again once it is installed)")
+ vision = "none"
ok("engine: the engine component (looked up when the model loads)")
+ ok("images: " + {"none": "off", "gpu": "on (read on the Mac's GPU)", "cpu": "on (read on the CPU)"}[vision])
step(5, f"downloading {fam['title']} {model}")
models_dir = Path(a.gguf_dir) if a.gguf_dir else models_root / tag
@@ -7033,6 +7069,18 @@ def mac_control_setup(a) -> int:
download(fam["hf"].format(q=model) + s.name, s)
check_shards(files)
ok("model files present")
+ mmproj = None
+ if vision != "none":
+ mmproj = Path(models_root) / fam["mmproj"]
+ if not mmproj.exists(): # an earlier install's copy (the same file for every family)
+ mmproj = find_in(roots, f"models/{fam['mmproj']}") or mmproj
+ if not mmproj.exists() and a.gguf_dir and (Path(a.gguf_dir) / fam["mmproj"]).exists():
+ mmproj = Path(a.gguf_dir) / fam["mmproj"]
+ elif not (mmproj.exists() and done(mmproj)):
+ download(fam["mmproj_hf"] + fam["mmproj"], mmproj, "vision encoder")
+ if fam["mmproj"] in ISTA_FILES:
+ verify_sha256(mmproj, *ISTA_FILES[fam["mmproj"]])
+ ok(f"vision encoder: {mmproj}")
step(6, "preparing the model for Strata")
pack = find_in(roots, f"packs/{tag.lower()}") or data / "packs" / tag.lower()
@@ -7072,6 +7120,13 @@ def mac_control_setup(a) -> int:
cfg["draft_vocab"] = a.draft_vocab
if a.browser is not None:
cfg["open_browser"] = a.browser
+ if vision != "none": # the Mac's one program reads on the GPU (--gpu) or the CPU (strata_control/vision_reader.py)
+ from strata_control import vision_reader as VR
+ cfg["args"].append("--vision")
+ cfg["vision"] = VR.vision_section(reader, {"exe": str(reader.exe), "mmproj": str(mmproj), "model": str(files[0]),
+ "gpu": vision == "gpu", "max_tokens": vision_tokens(a.vision_tokens, vision, None)})
+ if vision == "cpu":
+ cfg["vision"]["threads"] = vision_threads()
cfg_path = home() / f"strata-{tag.lower()}.json"
write_setup_config(cfg_path, cfg)
script = write_run_script(tag, cfg_path, port, cfg.get("open_browser") is not False)
diff --git a/strata_cli/launch/catalog.py b/strata_cli/launch/catalog.py
index 33041493b..efd52685b 100644
--- a/strata_cli/launch/catalog.py
+++ b/strata_cli/launch/catalog.py
@@ -420,12 +420,18 @@ class Catalog:
def picture_default(self, images: bool) -> str:
"""The picture setting nobody chose: off for a model that cannot read pictures or an engine without the reader; else on, with the
reader the one rule of strata_control/vision_reader.py gives: the CPU reader under 24 GB of graphics memory, the GPU reader from 24 GB,
- on every make of card (owner decision 2026-10-18). Without a usable graphics card there is nothing to read on: the CPU reader."""
+ on every make of card (owner decision 2026-10-18); on a Mac the Metal reader (unified memory). Without a usable graphics card there is nothing to read on: the CPU reader."""
from strata_control import vision_reader as VR
if not images or self.reader_state() == "missing":
return "off"
+ if self.is_mac():
+ return VR.default_mode(None, unified=True) # a Mac (unified memory): the Metal reader, nothing to keep free for a card
vram = self.computer().main_vram_gb
- return VR.default_mode(vram) if vram > 0 else "cpu" # a Mac (unified memory) or no card: nothing to keep free, the CPU reader
+ return VR.default_mode(vram) if vram > 0 else "cpu" # no usable card: nothing to read on, the CPU reader
+
+ def is_mac(self) -> bool:
+ """An Apple Silicon Mac: the chip's memory is the system's (the Metal engine; strata_control/vision_reader.py)."""
+ return self.engine_backend() == "metal"
# ---- installed models
def installed_configs(self) -> list[dict]:
diff --git a/strata_control/vision_reader.py b/strata_control/vision_reader.py
index 143445fe9..8678d5e32 100644
--- a/strata_control/vision_reader.py
+++ b/strata_control/vision_reader.py
@@ -1,4 +1,4 @@
-"""Which picture reader an engine folder gets: the GPU one (`strata-vision-hip`, AMD) or the CPU one (`strata-vision`).
+"""Which picture reader an engine folder gets: the GPU one (`strata-vision-hip`, AMD; `strata-vision --gpu` on a Mac) or the CPU one (`strata-vision`).
An AMD engine package carries two programs for pictures. `strata-vision` is the CPU reader: it needs nothing from the GPU
runtime, so it starts on every PC. `strata-vision-hip` is the same reader on the card (llama.cpp's HIP backend), a
@@ -7,6 +7,12 @@ reader as its fallback (`vision.cpu_exe`); the server (serve/server.py, Vision)
log line, when the GPU reader does not start: the HIP runtime does not load, the card is not one the build has code for
(the reader says so before it runs a kernel), or the card is busy. Nothing is asked of the user.
+A Mac has one program for both: `strata-vision` is built with Metal, `--gpu` reads on the Mac's GPU and without it on the CPU, so the
+CPU reader is the same file and the fallback needs no second program. Unified memory changes the 24 GB rule: nothing is taken from a
+separate card, the reader's ~1.1 GB is the mmproj file itself on either side (measured on an M4, footprint 1,050 MB on the CPU, 1,179 MB
+on the GPU), and the CPU reader takes ~50 s for a 260-token picture where the GPU one takes 0.45 s (ggml has no BF16 vector kernel on
+ARM). So a Mac reads on the GPU unless the person asks for the CPU.
+
`choose` is the whole decision; setup.py calls it, and tools/test_vision_reader.py checks every branch.
"""
from __future__ import annotations
@@ -24,13 +30,16 @@ GPU_READER = "strata-vision-hip"
GPU_MIN_VRAM_GB = 23.5
-def default_mode(vram_gb: float | None) -> str:
- """"cpu" or "gpu": the reader an unspecific "pictures on" gets on a card with this much memory (unknown memory: "gpu")."""
+def default_mode(vram_gb: float | None, *, unified: bool = False) -> str:
+ """"cpu" or "gpu": the reader an unspecific "pictures on" gets on a card with this much memory (unknown memory: "gpu").
+ unified: a Mac, where the chip's memory is the system's - always "gpu" (see the module doc)."""
+ if unified:
+ return "gpu"
vram = float(vram_gb or 0.0)
return "cpu" if 0 < vram < GPU_MIN_VRAM_GB else "gpu"
-def wanted(asked: str | None, vram_gb: float | None) -> str:
+def wanted(asked: str | None, vram_gb: float | None, *, unified: bool = False) -> str:
"""What is wanted for `asked` (None, "yes", "on", "gpu", "cpu", "no"/"none"/"off"): "gpu", "cpu" or "none". Nothing asked and "yes" mean
the default for this card; "gpu" and "cpu" are kept as asked."""
a = (asked or "").lower()
@@ -38,7 +47,7 @@ def wanted(asked: str | None, vram_gb: float | None) -> str:
return "none"
if a in ("gpu", "cpu"):
return a
- return default_mode(vram_gb)
+ return default_mode(vram_gb, unified=unified)
GFX = re.compile(r"gfx[0-9a-f]+", re.IGNORECASE)
@@ -47,13 +56,16 @@ def reader_name(base: str, windows: bool) -> str:
return base + (".exe" if windows else "")
-def find(folder: Path | str, windows: bool) -> dict:
- """The readers in an engine folder: {"cpu": Path | None, "gpu": Path | None}."""
+def find(folder: Path | str, windows: bool, *, mac: bool = False) -> dict:
+ """The readers in an engine folder: {"cpu": Path | None, "gpu": Path | None}. On a Mac (mac=True) `strata-vision` is both: it has
+ Metal built in and `--gpu` picks the GPU."""
folder = Path(folder)
found = {}
for key, base in (("cpu", CPU_READER), ("gpu", GPU_READER)):
path = folder / reader_name(base, windows)
found[key] = path if path.is_file() else None
+ if mac:
+ found["gpu"] = found["cpu"]
return found
@@ -71,16 +83,27 @@ class Reader:
note: str = "" # one plain sentence for the setup output when it is not what was asked for
-def choose(folder: Path | str, asked: str, *, windows: bool, gpu_arch: str = "", archs: list | None = None) -> Reader:
+MAC_CPU_NOTE = ("the picture reader on the CPU takes about 50 seconds for one picture on a Mac (the GPU reader: under a second): kept as "
+ "you chose; --vision gpu reads on the Mac's GPU")
+
+
+def choose(folder: Path | str, asked: str, *, windows: bool, gpu_arch: str = "", archs: list | None = None, mac: bool = False) -> Reader:
"""The reader for `asked` ("yes"/"on"/"gpu" = the best the folder has, "cpu", "no"/"none"/"off").
gpu_arch: the card's architecture when setup knows it ("gfx1201"); archs: what the GPU reader was built for
(BUILD.json `vision_archs`, else the package's `archs`). A card the build does not list gets the CPU reader here;
- when the card is unknown the GPU reader is tried and falls back by itself (see the module doc)."""
+ when the card is unknown the GPU reader is tried and falls back by itself (see the module doc).
+ mac: the Mac's one program, `strata-vision`: "gpu" runs it with --gpu and keeps the same file as the CPU reader to fall back on."""
asked = (asked or "").lower()
if asked in ("no", "none", "off", ""):
return Reader("none")
- have = find(folder, windows)
+ have = find(folder, windows, mac=mac)
+ if mac:
+ if have["cpu"] is None:
+ return Reader("none", note="this engine has no picture reader: pictures off")
+ if asked == "cpu":
+ return Reader("cpu", have["cpu"], note=MAC_CPU_NOTE)
+ return Reader("gpu", have["gpu"], have["cpu"])
if asked == "cpu":
if have["cpu"]:
return Reader("cpu", have["cpu"])
diff --git a/tests/terminal/test_launch.py b/tests/terminal/test_launch.py
index 39e545290..b07b4b7e5 100644
--- a/tests/terminal/test_launch.py
+++ b/tests/terminal/test_launch.py
@@ -333,6 +333,23 @@ class SetupArgs(unittest.TestCase):
off = Runner(S.LaunchSettings(family="qwen", model="IQ3_XXS", context=32768, vision="off"), catalog=catalog(NV))
self.assertEqual(off.setup_args()[off.setup_args().index("--vision") + 1], "no")
+ def test_a_mac_defaults_pictures_to_the_metal_reader_whatever_its_memory(self):
+ for gb in (16.0, 32.0, 128.0):
+ hw = {"ram_gb": gb, "os": "Darwin", "cpu": {"avx2": None, "name": "Apple M4"},
+ "mac": {"chip": "Apple M4", "macos": "27.2", "unified_gb": gb},
+ "gpus": [{"index": 0, "name": "Apple M4", "vendor": "apple", "vram_gb": 0.0, "unified_gb": gb, "usable": True}]}
+ with Home():
+ cat = catalog(hw)
+ with mock.patch.object(type(cat), "reader_state", lambda self: "present"):
+ self.assertTrue(cat.is_mac())
+ self.assertEqual(cat.picture_default(True), "gpu", gb)
+ r = Runner(S.LaunchSettings(family="qwen", model="Q2_0", context=32768), catalog=cat)
+ with mock.patch.object(sys, "platform", "darwin"):
+ args = r.setup_args()
+ self.assertEqual(args[args.index("--vision") + 1], "gpu", gb)
+ with mock.patch.object(type(cat), "reader_state", lambda self: "missing"):
+ self.assertEqual(cat.picture_default(True), "off")
+
def test_a_model_that_cannot_read_pictures_stays_off_without_a_question(self):
import dataclasses
with Home():
diff --git a/tools/strata_mcp.py b/tools/strata_mcp.py
index 8780210ac..f9ddcc04f 100644
--- a/tools/strata_mcp.py
+++ b/tools/strata_mcp.py
@@ -985,11 +985,12 @@ class Strata:
d = models[m]
tag = (fd.get("tag", "") + m).lower()
sizes.append({"model": m, "about": d["about"], "download_gb": d["download_gb"], "experimental": False,
- "images": False, # the app's first release reads no pictures (setup.py's Control mode says so)
+ "images": d.get("vision", fd.get("vision", True)) is not False, # the Mac engine carries the picture reader (Metal)
"ram_needed_gb": d["ram_gb"], "experts_gb": d.get("arena_gb"), "mem_total": False,
"memory_gb": round(float(d.get("arena_gb") or d["ram_gb"]) + 1.0, 1),
"on_this_pc": self._mac_verdict(m, ram) if ram else None, "installed": tag in have, "id": tag})
- return [{"family": "qwen", "title": fd["title"], "about": fd.get("about"), "experimental": False, "images": False, "sizes": sizes}]
+ return [{"family": "qwen", "title": fd["title"], "about": fd.get("about"), "experimental": False,
+ "images": any(x["images"] for x in sizes), "sizes": sizes}]
def recommend(self, hw: dict) -> dict:
"""setup.py's own rules: the size it preselects (IQ3_XXS from 60 GB of RAM, else the first, Q2_0), the
@@ -1848,9 +1849,9 @@ class Tools:
raise ToolError(f"images are not available with {families[family]['title']} {model} yet: use vision=no")
if vision is None:
# owner rule (2026-10-18): pictures are on by default wherever the model can read them and the engine has a reader; setup picks
- # the reader (strata_control/vision_reader.py: the CPU under 24 GB, the GPU from 24 GB, every make of card). A Mac has none yet.
+ # the reader (strata_control/vision_reader.py: the CPU under 24 GB, the GPU from 24 GB, every make of card; a Mac reads on its GPU).
can_read = md.get("vision", families[family].get("vision")) is not False
- vision = "yes" if can_read and not mac else "no"
+ vision = "yes" if can_read else "no"
target = self.check_data_dir(data_dir) if data_dir else s.data_dir()
tag = (families[family].get("tag", "") + model)
have_dir = target / "models" / tag
diff --git a/tools/test_picture_defaults.py b/tools/test_picture_defaults.py
index 0e51f2729..8f141de52 100644
--- a/tools/test_picture_defaults.py
+++ b/tools/test_picture_defaults.py
@@ -71,6 +71,30 @@ class PictureDefaults(unittest.TestCase):
self.assertEqual(setup.hip_vision(None, 16.0), "cpu")
self.assertEqual(setup.hip_vision("gpu", 16.0), "gpu")
+class MacPictureDefaults(unittest.TestCase):
+ """setup.py on a Mac: pictures on by default, on the Metal reader (strata_control/vision_reader.py)."""
+
+ def test_nothing_asked_means_the_gpu_reader_for_every_mac_size(self):
+ from strata_control import mac_fit as MF
+ for model in MF.MAC_MODELS:
+ for ram in (32.0, 64.0, 128.0):
+ self.assertEqual(setup.mac_vision(None, model, ram), "gpu", (model, ram))
+ self.assertEqual(setup.mac_vision("yes", model, ram), "gpu", (model, ram))
+
+ def test_off_and_cpu_are_kept_as_asked(self):
+ for off in ("no", "none", "off"):
+ self.assertEqual(setup.mac_vision(off, "Q2_0", 32.0), "none")
+ self.assertEqual(setup.mac_vision("cpu", "Q2_0", 32.0), "cpu")
+
+ def test_a_folder_without_the_reader_turns_pictures_off(self):
+ import tempfile
+ d = Path(tempfile.mkdtemp(prefix="macvis-"))
+ (d / "strata").write_bytes(b"x")
+ self.assertEqual(setup.mac_reader(d, "gpu").mode, "none")
+ (d / "strata-vision").write_bytes(b"x")
+ r = setup.mac_reader(d, "gpu")
+ self.assertEqual((r.mode, r.exe.name), ("gpu", "strata-vision"))
+
if __name__ == "__main__":
unittest.main()
diff --git a/tools/test_strata_mcp.py b/tools/test_strata_mcp.py
index 089a286ad..1bcb31d1c 100644
--- a/tools/test_strata_mcp.py
+++ b/tools/test_strata_mcp.py
@@ -706,7 +706,7 @@ class MacCatalog(unittest.TestCase):
self.assertIn("slower", by["Q2_0"]["on_this_pc"])
self.assertIn("~14 tok/s", by["Q2_0"]["on_this_pc"])
self.assertTrue(by["IQ3_S"]["on_this_pc"].startswith("slow here"))
- self.assertFalse(any(x["images"] for x in by.values()))
+ self.assertTrue(all(x["images"] for x in by.values()), "the Mac engine carries the Metal picture reader: pictures are on")
s, hw = self.hw(128)
self.assertEqual({x["on_this_pc"] for x in s.model_table(hw)[0]["sizes"]}, {"fits"})
s, hw = self.hw(8)
@@ -723,6 +723,9 @@ class MacCatalog(unittest.TestCase):
for flag in ("--kv", "--gpu", "--gpus", "--low-ram", "--backend"):
self.assertNotIn(flag, args)
self.assertIn("--gguf-dir", args)
+ with tempfile.TemporaryDirectory() as own: # nothing said about pictures: on, as everywhere (the Mac's reader is the Metal one)
+ _plan, args = M.Tools(s).install_plan("qwen", "Q2_0", None, None, "int8", 0, None, None, "auto", None, None, own)
+ self.assertEqual(args[args.index("--vision") + 1], "yes")
with self.assertRaises(M.ToolError):
s2, hw2 = self.hw(8)
s2.hardware = lambda fresh=False: hw2
diff --git a/tools/test_vision_reader.py b/tools/test_vision_reader.py
index e840aafac..d05739fe8 100644
--- a/tools/test_vision_reader.py
+++ b/tools/test_vision_reader.py
@@ -104,6 +104,48 @@ class Choose(unittest.TestCase):
self.assertIsNone(VR.cpu_fallback({"exe": "strata-vision", "gpu": True})) # an NVIDIA config has no fallback
+class MacReader(unittest.TestCase):
+ """A Mac has one picture program, strata-vision with Metal built in: --gpu reads on the GPU, without it on the CPU."""
+
+ def test_a_mac_reads_on_the_gpu_by_default_whatever_its_memory(self):
+ for ram in (16, 32, 64, 128, None):
+ self.assertEqual(VR.default_mode(ram, unified=True), "gpu", ram)
+ self.assertEqual(VR.wanted(None, ram, unified=True), "gpu", ram)
+ self.assertEqual(VR.wanted("yes", ram, unified=True), "gpu", ram)
+ self.assertEqual(VR.wanted("cpu", 32, unified=True), "cpu") # asked for: kept
+ self.assertEqual(VR.wanted("off", 32, unified=True), "none")
+ self.assertEqual(VR.default_mode(16), "cpu") # a 16 GB card of a PC still follows the 24 GB rule
+
+ def test_the_one_program_is_both_readers_and_its_own_fallback(self):
+ d = folder("strata", "strata-vision")
+ for ask in ("yes", "on", "gpu"):
+ r = VR.choose(d, ask, windows=False, mac=True)
+ self.assertEqual((r.mode, r.exe.name, r.cpu_exe.name, r.note), ("gpu", "strata-vision", "strata-vision", ""), ask)
+ sec = VR.vision_section(VR.choose(d, "yes", windows=False, mac=True), {"mmproj": "m.gguf", "model": "t.gguf", "max_tokens": 1024})
+ self.assertTrue(sec["gpu"])
+ self.assertEqual(sec["exe"], sec["cpu_exe"])
+ fallback = VR.cpu_fallback(sec)
+ self.assertFalse(fallback["gpu"])
+ self.assertEqual(fallback["exe"], sec["exe"])
+
+ def test_cpu_asked_is_kept_with_the_time_it_takes(self):
+ r = VR.choose(folder("strata-vision"), "cpu", windows=False, mac=True)
+ self.assertEqual((r.mode, r.exe.name, r.cpu_exe), ("cpu", "strata-vision", None))
+ self.assertIn("50 seconds", r.note)
+ self.assertFalse(VR.vision_section(r, {})["gpu"])
+
+ def test_an_engine_without_the_reader_gets_pictures_off(self):
+ for ask in ("yes", "gpu", "cpu"):
+ r = VR.choose(folder("strata"), ask, windows=False, mac=True)
+ self.assertEqual(r.mode, "none", ask)
+ self.assertIn("no picture reader", r.note)
+ self.assertEqual(VR.choose(folder("strata-vision"), "off", windows=False, mac=True).mode, "none")
+
+ def test_a_hip_program_is_not_a_mac_reader(self):
+ r = VR.choose(folder("strata-vision-hip"), "yes", windows=False, mac=True)
+ self.assertEqual(r.mode, "none")
+
+
class FakeProc:
def __init__(self, first_line: str):
self.stdin, self.stdout = io.StringIO(), io.StringIO(first_line)
diff --git a/tools/vision/CMakeLists.txt b/tools/vision/CMakeLists.txt
index 5f031b9ce..e2f958d4f 100644
--- a/tools/vision/CMakeLists.txt
+++ b/tools/vision/CMakeLists.txt
@@ -23,6 +23,7 @@ endif()
if(STRATA_VISION_HIP AND NOT STRATA_VISION_HIP_ARCHS)
message(FATAL_ERROR "strata-vision: STRATA_VISION_HIP needs STRATA_VISION_HIP_ARCHS, for example \"gfx1200;gfx1201\"")
endif()
+option(STRATA_VISION_METAL "a Mac: ggml's Metal backend in the program, so --gpu reads pictures on the Mac's GPU (OFF: CPU only)" ON)
option(STRATA_PORTABLE "build for other PCs: AVX2 baseline, static MSVC runtime" ON)
if(STRATA_PORTABLE AND MSVC)
set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded")
@@ -62,6 +63,15 @@ endif()
set(GGML_BACKEND_DL OFF CACHE BOOL "" FORCE)
set(GGML_CUDA ${STRATA_VISION_CUDA} CACHE BOOL "" FORCE)
set(GGML_HIP ${STRATA_VISION_HIP} CACHE BOOL "" FORCE)
+if(APPLE)
+ # One program for both readers: strata-vision --gpu runs the encoder on the Metal GPU, without --gpu on the CPU (Apple Silicon always
+ # has the Metal framework, the engine needs it too). The Metal shaders are embedded in the binary and compiled by macOS when the
+ # reader starts, so nothing but the program has to be shipped. No -march flag: the default target (an M1) runs on every Apple
+ # Silicon Mac, and ggml's CPU path has no BF16 vector kernel on ARM (measured on an M4: the same ~50 s for a 260-token picture with
+ # -march=armv8.6-a+bf16+i8mm as without), so a faster target would only risk the M1.
+ set(GGML_METAL ${STRATA_VISION_METAL} CACHE BOOL "" FORCE)
+ set(GGML_METAL_EMBED_LIBRARY ${STRATA_VISION_METAL} CACHE BOOL "" FORCE)
+endif()
if(STRATA_VISION_HIP)
# the engine's HIP build says the same (cmake/hip_backend.cmake); ggml-hip reads AMDGPU_TARGETS / CMAKE_HIP_ARCHITECTURES
set(AMDGPU_TARGETS "${STRATA_VISION_HIP_ARCHS}" CACHE STRING "" FORCE)
@@ -82,6 +92,9 @@ if(STRATA_VISION_HIP)
string(REPLACE ";" "," _hip_archs "${STRATA_VISION_HIP_ARCHS}")
target_compile_definitions(strata-vision PRIVATE STRATA_VISION_HIP STRATA_VISION_HIP_ARCHS="${_hip_archs}")
endif()
+if(APPLE AND STRATA_VISION_METAL)
+ target_compile_definitions(strata-vision PRIVATE STRATA_VISION_METAL)
+endif()
if(WIN32 AND STRATA_VISION_HIP)
# hipblas.dll (and rocblas.dll behind it) live in rocm\bin of the engine package: loaded on first use, after main() has put that folder on
# the DLL search path, so the reader starts from any PATH and says "no GPU" itself where there is none
diff --git a/tools/vision/strata_vision.cpp b/tools/vision/strata_vision.cpp
index eef3e81c9..fb892630c 100644
--- a/tools/vision/strata_vision.cpp
+++ b/tools/vision/strata_vision.cpp
@@ -102,6 +102,8 @@ int main(int argc, char** argv) {
else if (a == "--version") {
#ifdef STRATA_VISION_HIP
std::printf("strata-vision hip (GPU architectures: %s)\n", STRATA_VISION_HIP_ARCHS);
+#elif defined(STRATA_VISION_METAL)
+ std::printf("strata-vision metal\n");
#else
std::printf("strata-vision\n");
#endif
@@ -162,6 +164,23 @@ int main(int argc, char** argv) {
}
}
#endif
+#ifdef STRATA_VISION_METAL
+ if (gpu) {
+ // The Mac: the same program reads on the GPU (--gpu, ggml's Metal backend) or on the CPU. With --gpu and no Metal device (a
+ // virtual machine without a GPU, say), mtmd would quietly use the CPU, which reads a picture in ~50 s: say so instead, and
+ // the server starts the program again without --gpu (serve/server.py, Vision._start) with one log line.
+ bool have_gpu = false;
+ for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
+ const auto type = ggml_backend_dev_type(ggml_backend_dev_get(i));
+ have_gpu = have_gpu || type == GGML_BACKEND_DEVICE_TYPE_GPU || type == GGML_BACKEND_DEVICE_TYPE_IGPU;
+ }
+ if (!have_gpu) {
+ std::printf("ERR this Mac has no Metal GPU for the picture reader\n");
+ std::fflush(stdout);
+ return 3;
+ }
+ }
+#endif
const auto load_t0 = std::chrono::steady_clock::now();
#if !defined(_WIN32) && !defined(__APPLE__) // macOS has no posix_fadvise