Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Intel Arc: pictures on by default through the portable CPU reader (setup_intel, shared /dev/shm for the embeddings)

Niko1221667d3cf7d785
diff --git a/.gitignore b/.gitignore
index 90bec59ad..c6cfb3367 100644
--- a/.gitignore
+++ b/.gitignore
@@ -48,6 +48,7 @@ build-vision-cuda12.bat
# setup.py outputs (engine, downloads) and release builds
/engine/
+/engine-sycl/
/engine-cuda12/
/build-release*/
/dist/
diff --git a/docs/INTEL.md b/docs/INTEL.md
index 77fa10ef0..8d42e9f9f 100644
--- a/docs/INTEL.md
+++ b/docs/INTEL.md
@@ -772,8 +772,11 @@ is 0.012), first-token KL 0.0017 (floor 0.0005), top-1 agreement 99.7%. HumanEva
## Not done
-- Images, on both Intel engines. Strata's vision path encodes with `strata-vision` into embeddings the CUDA
- engine reads; neither the SYCL port nor the llama.cpp config wires it yet.
+- Images on the llama.cpp config: not wired. On the SYCL port pictures are on by default (0.2.0): `setup --backend sycl` compiles the
+ portable CPU reader `strata-vision` (the program the AMD packages carry beside their engine) into `engine-sycl/`, the server runs
+ it on the host, and the engine in the container reads its embeddings from the host's `/dev/shm`, which `strata-sycl.sh` shares
+ (`STRATA_SYCL_SHM=0` keeps a private one). `--vision no` turns pictures off. A SYCL GPU reader does not exist yet, so every Arc
+ reads pictures on the CPU (a few seconds each), whatever its memory; that is later work.
- Speculative decoding on the llama.cpp path (the GGUF carries no draft layer llama.cpp can use). The SYCL
port has it (MTP draft layer, `--mtp`).
diff --git a/sycl/serve/server_intel.py b/sycl/serve/server_intel.py
index baae26c23..8310bc761 100644
--- a/sycl/serve/server_intel.py
+++ b/sycl/serve/server_intel.py
@@ -47,6 +47,24 @@ def install_xe_reader():
T.gpu_reader = gpu_reader
+def install_shared_embeddings():
+ """Pictures: the engine runs in a container, and a request's image embeddings reach it as a file named on its GENI line.
+ serve/server.py writes that file to /dev/shm when that has room, and strata-sycl.sh shares the host's /dev/shm with the
+ container, so the path means the same on both sides. Anywhere else (a full /dev/shm) the container cannot see the file:
+ said once, in the log, in plain words."""
+ orig = S.combined_embeddings_path
+ said = []
+
+ def path(vision_dir, nbytes):
+ p = orig(vision_dir, nbytes)
+ if not str(p).startswith("/dev/shm/") and not said:
+ said.append(1)
+ print("[strata] /dev/shm has no room for this picture: the engine runs in a container that sees only the host's /dev/shm, so it "
+ "cannot read the picture. Free /dev/shm (or enlarge it: mount -o remount,size=4G /dev/shm) and send it again.", flush=True)
+ return p
+ S.combined_embeddings_path = path
+
+
def model_switcher(url: str, port, model, mode: str = "") -> dict:
"""The host's model swapper: the models it serves on THIS port, which one holds the card, whether one is loading.
With `mode`, asks for that one first; this server is then usually the one stopped, so the web app waits for
@@ -308,6 +326,7 @@ def top_logprobs(req: dict):
if __name__ == "__main__":
install_xe_reader()
+ install_shared_embeddings()
install_narrator()
install_switcher(sys.argv[1:])
install_logprobs()
diff --git a/sycl/serve/strata-sycl.sh b/sycl/serve/strata-sycl.sh
index f579e79c9..2f2120a3f 100755
--- a/sycl/serve/strata-sycl.sh
+++ b/sycl/serve/strata-sycl.sh
@@ -7,6 +7,9 @@
# STRATA_SYCL_IMAGE the runtime image (default: strata-sycl-dev)
# STRATA_SYCL_BIN the engine binary, relative to the repo (default: build-sycl-aot/strata)
# STRATA_SYCL_NAME the container's name (default: strata-sycl-serve)
+# STRATA_SYCL_SHM 1 (default): the host's /dev/shm is the container's /dev/shm. Pictures: the server (on the host) writes the
+# image embeddings of a request into /dev/shm and names that file on the engine's GENI line, so the engine in
+# the container must see the same file. 0: the container keeps a private /dev/shm.
# Passed into the container: every variable of this script's environment that starts with STRATA_ (the config's
# "env" block arrives that way: the server puts it in the engine's environment), ONEAPI_ (level_zero:* for a
# two-card split, #423), UR_, IGC_, SYCL_ or ZES_. The port's own defaults (below) are overridden the same way,
@@ -34,9 +37,12 @@ for k in "${!setting[@]}"; do
# the engine tests the switches by presence: a value of 0 (or empty) means "not set", so it is not passed on
case "${setting[$k]}" in 0|"") ;; *) envs+=(-e "$k=${setting[$k]}") ;; esac
done
+shm=()
+if [ "${STRATA_SYCL_SHM:-1}" != 0 ] && [ -d /dev/shm ]; then shm=(-v /dev/shm:/dev/shm); fi
exec docker run --rm -i --name "$name" --device /dev/dri --oom-score-adj 1000 --stop-timeout 30 --no-healthcheck \
--ulimit memlock=-1 \
-v "$root:/work" \
+ "${shm[@]}" \
"${envs[@]}" \
"${STRATA_SYCL_IMAGE:-strata-sycl-dev}" \
"cd $repo_in && exec ${STRATA_SYCL_BIN:-build-sycl-aot/strata}$args"
diff --git a/sycl/setup_intel.py b/sycl/setup_intel.py
index 07a791012..fca5dc8d6 100644
--- a/sycl/setup_intel.py
+++ b/sycl/setup_intel.py
@@ -7,10 +7,14 @@ it and replaces the few steps that are NVIDIA/AMD-specific, then runs setup's ow
model choice, the download, the pack, the tokenizer, the MTP draft layer, the context and KV questions - is
setup.py's, unchanged. What is replaced:
- - the GPU check: the Intel Arc is offered through setup's AMD path (the one that compiles locally and has no
- images), named and sized from sysfs;
+ - the GPU check: the Intel Arc is offered through setup's AMD path (the one that compiles locally), named and
+ sized from sysfs;
- the engine step: the SYCL build (build-sycl-aot/strata, run in the strata-sycl-dev image by
sycl/serve/strata-sycl.sh; docs/INTEL.md) instead of a CUDA/HIP build;
+ - pictures: on by default, read by the portable CPU reader (`strata-vision`, the program the AMD packages carry beside
+ their engine, compiled here by setup's own step) on the host; it hands the image embeddings to the engine through
+ /dev/shm, which strata-sycl.sh shares with the container. A SYCL GPU reader does not exist yet, so "gpu" is read as "cpu"
+ here; --vision no is the off switch;
- the RAM rule: the CUDA engine keeps every expert in RAM, the SYCL port streams them from the GGUF into VRAM
(--stream-experts), so RAM only bounds the KV streaming;
- the config: the container's paths, the SYCL flags, backend "sycl"; the run script starts
@@ -20,14 +24,11 @@ When setup.py changes the steps this relies on, this stops with a message rather
"""
from __future__ import annotations
-import atexit
import json
import os
import re
-import shutil
import subprocess
import sys
-import tempfile
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
@@ -121,6 +122,15 @@ def drop(args, name, value=False):
del args[i:i + 1 + int(value)]
+READER_DIR = ROOT / "engine-sycl" # the folder setup treats as the engine's: BUILD.json and, for pictures, the CPU reader strata-vision
+
+
+def intel_vision(asked, vram_gb: float = 0.0) -> str:
+ """The picture reader wanted on an Intel Arc: "none" when switched off, else "cpu" (the only reader the SYCL port has; a
+ SYCL GPU reader is later work). Takes setup.hip_vision's arguments."""
+ return "none" if str(asked or "").lower() in ("no", "none", "off") else "cpu"
+
+
SYCL_PROFILE = "expert-profile-sycl.bin" # data/: the xe cards' expert ranking (to_sycl)
SMALL_CARD_GB = 12.0 # under this the engine's own advice is --vram-reserve-mib 300 (a 1024 MiB reserve leaves an
# 8 GB card no room for even one expert slot) and the English draft vocabulary
@@ -222,7 +232,7 @@ def to_sycl(cfg: dict, exe: Path, ram: float, keep: dict, vram_gb: float = 0.0,
if flag(args, "--prefill") in (None, "auto", "2048", "4096", "8192"):
drop(args, "--prefill", True)
args += ["--prefill", str(recommended_prefill(vram_gb, ctx, driver, devid)[0])]
- out = {k: v for k, v in cfg.items() if k not in ("lib_dirs", "env", "vision", "gpus")}
+ out = {k: v for k, v in cfg.items() if k not in ("lib_dirs", "env", "gpus")} # "vision" stays: the CPU reader runs on the host
out.update({"backend": "sycl", "exe": str(SYCL_WRAPPER), "args": args, "sycl_root": str(MOUNT)})
env = {}
if alchemist:
@@ -264,9 +274,13 @@ def install(argv) -> None:
keep[p.name] = {k: c[k] for k in ("model_switcher", "sampling") if k in c}
except (OSError, ValueError):
pass
- stub = Path(tempfile.mkdtemp(prefix="strata-sycl-")) # setup reads the engine's BUILD.json; the SYCL build has none
- atexit.register(shutil.rmtree, stub, True)
- (stub / "BUILD.json").write_text(json.dumps({"version": sycl_version(), "source": "local", "lib_dirs": []}))
+ stub = READER_DIR # setup reads the engine's BUILD.json; the SYCL build has none. A folder that
+ stub.mkdir(exist_ok=True) # stays: the picture reader in it is what the run config points at
+ try:
+ stamped = json.loads((stub / "BUILD.json").read_text(encoding="utf-8")) # keeps what the reader's build recorded
+ except (OSError, ValueError):
+ stamped = {}
+ (stub / "BUILD.json").write_text(json.dumps({**stamped, "version": sycl_version(), "source": "local", "lib_dirs": []}))
say = S.say
fake_ram = max(real_ram, 1024.0)
@@ -294,8 +308,15 @@ def install(argv) -> None:
S.gpus = lambda *a, **k: []
S.amd_gpus = lambda *a, **k: intel
S.amd_problem = lambda g: None
- S.hip_vision = lambda asked, vram_gb=0.0: "none" # images are not wired on the SYCL port yet
- S.build_engine_hip = lambda *a, **k: stub
+ S.hip_vision = intel_vision # pictures on by default, read on the CPU (setup's hip_reader settles it)
+
+ def build_engine_hip(gpu, llama, vision="none"):
+ """No engine to compile here (the SYCL build is separate); the picture reader is: the portable CPU reader, once."""
+ if vision != "none":
+ stamp = stub / "BUILD.json"
+ S.build_vision_cpu(stub, stamp, json.loads(stamp.read_text(encoding="utf-8")), llama, S.source_hash(S.VISION_SOURCES))
+ return stub
+ S.build_engine_hip = build_engine_hip
S.hipblaslt_table = lambda *a, **k: None
S.ram_gb = lambda: fake_ram # the experts are in VRAM: setup's RAM rule does not apply
@@ -376,8 +397,6 @@ def install(argv) -> None:
sys.argv = [str(ROOT / "setup.py"), *argv]
if "--backend" not in argv:
sys.argv += ["--backend", "hip"]
- if "--vision" not in argv:
- sys.argv += ["--vision", "none"]
if "--low-ram" not in argv:
sys.argv += ["--low-ram", "off"]
sys.exit(S.main())
diff --git a/tools/test_setup_sycl.py b/tools/test_setup_sycl.py
index 1c993a6ae..2f3ad0759 100644
--- a/tools/test_setup_sycl.py
+++ b/tools/test_setup_sycl.py
@@ -223,6 +223,53 @@ class ToSycl(unittest.TestCase):
self.assertTrue(m.small_card(10.0, "xe"))
self.assertFalse(m.small_card(32.0, "xe"))
+ def test_pictures_are_on_by_default_read_on_the_cpu_and_the_vision_section_survives(self):
+ """Intel Arc: the CPU reader (strata-vision) on the host, --vision no is the off switch; "gpu" is read as "cpu" (no SYCL GPU reader)."""
+ m = self.mod()
+ for asked in (None, "yes", "gpu", "cpu", "on"):
+ self.assertEqual(m.intel_vision(asked, 8.0), "cpu")
+ self.assertEqual(m.intel_vision(asked, 32.0), "cpu")
+ for asked in ("no", "none", "off"):
+ self.assertEqual(m.intel_vision(asked, 32.0), "none")
+ section = {"exe": "/h/engine-sycl/strata-vision", "mmproj": "/h/models/mmproj.gguf", "model": "/h/models/m.gguf", "gpu": False,
+ "max_tokens": 300}
+ cfg = self.cfg(m, ["--vision"])
+ cfg["vision"] = section
+ c = m.to_sycl(cfg, m.ROOT / "build-sycl-aot" / "strata", 64.0, {}, 32.0, "xe")
+ self.assertEqual(c["vision"], section) # host paths: the reader runs on the host, not in the container
+ self.assertIn("--vision", c["args"])
+
+ def test_setup_intel_no_longer_forces_pictures_off(self):
+ src = (ROOT / "sycl" / "setup_intel.py").read_text(encoding="utf-8")
+ self.assertNotIn('sys.argv += ["--vision", "none"]', src)
+ self.assertIn("S.hip_vision = intel_vision", src)
+ self.assertIn("S.build_vision_cpu(", src) # the CPU reader is compiled by setup's own step
+
+ def test_the_container_shares_dev_shm_with_the_server_for_the_embeddings(self):
+ sh = (ROOT / "sycl" / "serve" / "strata-sycl.sh").read_text(encoding="utf-8")
+ self.assertIn("-v /dev/shm:/dev/shm", sh)
+ self.assertIn("STRATA_SYCL_SHM", sh)
+
+ def test_a_picture_that_cannot_reach_the_container_is_said_once(self):
+ import importlib.util
+ spec = importlib.util.spec_from_file_location("server_intel_t", ROOT / "sycl" / "serve" / "server_intel.py")
+ src = (ROOT / "sycl" / "serve" / "server_intel.py").read_text(encoding="utf-8")
+ self.assertIn("install_shared_embeddings()", src.split('if __name__ == "__main__":')[1])
+ sys.path.insert(0, str(ROOT / "sycl" / "serve"))
+ try:
+ m = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(m)
+ except ImportError as e: # no oneAPI / xe telemetry import here: the source check above stands
+ self.skipTest(str(e))
+ finally:
+ sys.path.pop(0)
+ out = io.StringIO()
+ with mock.patch.object(m.S, "combined_embeddings_path", lambda d, n: Path("/tmp/x/req.sve")), contextlib.redirect_stdout(out):
+ m.install_shared_embeddings()
+ m.S.combined_embeddings_path(Path("/tmp/x"), 10)
+ m.S.combined_embeddings_path(Path("/tmp/x"), 10)
+ self.assertEqual(out.getvalue().count("/dev/shm has no room"), 1)
+
if __name__ == "__main__":
unittest.main()