Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Intel Arc: pictures on by default through the portable CPU reader (setup_intel, shared /dev/shm for the embeddings)
Niko1221667d3cf7d785
diff --git a/.gitignore b/.gitignoreindex 90bec59ad..c6cfb3367 100644--- a/.gitignore+++ b/.gitignore@@ -48,6 +48,7 @@ build-vision-cuda12.bat# setup.py outputs (engine, downloads) and release builds/engine/+/engine-sycl//engine-cuda12//build-release*//dist/diff --git a/docs/INTEL.md b/docs/INTEL.mdindex 77fa10ef0..8d42e9f9f 100644--- a/docs/INTEL.md+++ b/docs/INTEL.md@@ -772,8 +772,11 @@ is 0.012), first-token KL 0.0017 (floor 0.0005), top-1 agreement 99.7%. HumanEva## Not done-- Images, on both Intel engines. Strata's vision path encodes with `strata-vision` into embeddings the CUDA- engine reads; neither the SYCL port nor the llama.cpp config wires it yet.+- Images on the llama.cpp config: not wired. On the SYCL port pictures are on by default (0.2.0): `setup --backend sycl` compiles the+ portable CPU reader `strata-vision` (the program the AMD packages carry beside their engine) into `engine-sycl/`, the server runs+ it on the host, and the engine in the container reads its embeddings from the host's `/dev/shm`, which `strata-sycl.sh` shares+ (`STRATA_SYCL_SHM=0` keeps a private one). `--vision no` turns pictures off. A SYCL GPU reader does not exist yet, so every Arc+ reads pictures on the CPU (a few seconds each), whatever its memory; that is later work.- Speculative decoding on the llama.cpp path (the GGUF carries no draft layer llama.cpp can use). The SYCLport has it (MTP draft layer, `--mtp`).diff --git a/sycl/serve/server_intel.py b/sycl/serve/server_intel.pyindex baae26c23..8310bc761 100644--- a/sycl/serve/server_intel.py+++ b/sycl/serve/server_intel.py@@ -47,6 +47,24 @@ def install_xe_reader():T.gpu_reader = gpu_reader+def install_shared_embeddings():+ """Pictures: the engine runs in a container, and a request's image embeddings reach it as a file named on its GENI line.+ serve/server.py writes that file to /dev/shm when that has room, and strata-sycl.sh shares the host's /dev/shm with the+ container, so the path means the same on both sides. Anywhere else (a full /dev/shm) the container cannot see the file:+ said once, in the log, in plain words."""+ orig = S.combined_embeddings_path+ said = []++ def path(vision_dir, nbytes):+ p = orig(vision_dir, nbytes)+ if not str(p).startswith("/dev/shm/") and not said:+ said.append(1)+ print("[strata] /dev/shm has no room for this picture: the engine runs in a container that sees only the host's /dev/shm, so it "+ "cannot read the picture. Free /dev/shm (or enlarge it: mount -o remount,size=4G /dev/shm) and send it again.", flush=True)+ return p+ S.combined_embeddings_path = path++def model_switcher(url: str, port, model, mode: str = "") -> dict:"""The host's model swapper: the models it serves on THIS port, which one holds the card, whether one is loading.With `mode`, asks for that one first; this server is then usually the one stopped, so the web app waits for@@ -308,6 +326,7 @@ def top_logprobs(req: dict):if __name__ == "__main__":install_xe_reader()+ install_shared_embeddings()install_narrator()install_switcher(sys.argv[1:])install_logprobs()diff --git a/sycl/serve/strata-sycl.sh b/sycl/serve/strata-sycl.shindex f579e79c9..2f2120a3f 100755--- a/sycl/serve/strata-sycl.sh+++ b/sycl/serve/strata-sycl.sh@@ -7,6 +7,9 @@# STRATA_SYCL_IMAGE the runtime image (default: strata-sycl-dev)# STRATA_SYCL_BIN the engine binary, relative to the repo (default: build-sycl-aot/strata)# STRATA_SYCL_NAME the container's name (default: strata-sycl-serve)+# STRATA_SYCL_SHM 1 (default): the host's /dev/shm is the container's /dev/shm. Pictures: the server (on the host) writes the+# image embeddings of a request into /dev/shm and names that file on the engine's GENI line, so the engine in+# the container must see the same file. 0: the container keeps a private /dev/shm.# Passed into the container: every variable of this script's environment that starts with STRATA_ (the config's# "env" block arrives that way: the server puts it in the engine's environment), ONEAPI_ (level_zero:* for a# two-card split, #423), UR_, IGC_, SYCL_ or ZES_. The port's own defaults (below) are overridden the same way,@@ -34,9 +37,12 @@ for k in "${!setting[@]}"; do# the engine tests the switches by presence: a value of 0 (or empty) means "not set", so it is not passed oncase "${setting[$k]}" in 0|"") ;; *) envs+=(-e "$k=${setting[$k]}") ;; esacdone+shm=()+if [ "${STRATA_SYCL_SHM:-1}" != 0 ] && [ -d /dev/shm ]; then shm=(-v /dev/shm:/dev/shm); fiexec docker run --rm -i --name "$name" --device /dev/dri --oom-score-adj 1000 --stop-timeout 30 --no-healthcheck \--ulimit memlock=-1 \-v "$root:/work" \+ "${shm[@]}" \"${envs[@]}" \"${STRATA_SYCL_IMAGE:-strata-sycl-dev}" \"cd $repo_in && exec ${STRATA_SYCL_BIN:-build-sycl-aot/strata}$args"diff --git a/sycl/setup_intel.py b/sycl/setup_intel.pyindex 07a791012..fca5dc8d6 100644--- a/sycl/setup_intel.py+++ b/sycl/setup_intel.py@@ -7,10 +7,14 @@ it and replaces the few steps that are NVIDIA/AMD-specific, then runs setup's owmodel choice, the download, the pack, the tokenizer, the MTP draft layer, the context and KV questions - issetup.py's, unchanged. What is replaced:- - the GPU check: the Intel Arc is offered through setup's AMD path (the one that compiles locally and has no- images), named and sized from sysfs;+ - the GPU check: the Intel Arc is offered through setup's AMD path (the one that compiles locally), named and+ sized from sysfs;- the engine step: the SYCL build (build-sycl-aot/strata, run in the strata-sycl-dev image bysycl/serve/strata-sycl.sh; docs/INTEL.md) instead of a CUDA/HIP build;+ - pictures: on by default, read by the portable CPU reader (`strata-vision`, the program the AMD packages carry beside+ their engine, compiled here by setup's own step) on the host; it hands the image embeddings to the engine through+ /dev/shm, which strata-sycl.sh shares with the container. A SYCL GPU reader does not exist yet, so "gpu" is read as "cpu"+ here; --vision no is the off switch;- the RAM rule: the CUDA engine keeps every expert in RAM, the SYCL port streams them from the GGUF into VRAM(--stream-experts), so RAM only bounds the KV streaming;- the config: the container's paths, the SYCL flags, backend "sycl"; the run script starts@@ -20,14 +24,11 @@ When setup.py changes the steps this relies on, this stops with a message rather"""from __future__ import annotations-import atexitimport jsonimport osimport re-import shutilimport subprocessimport sys-import tempfilefrom pathlib import PathROOT = Path(__file__).resolve().parent.parent@@ -121,6 +122,15 @@ def drop(args, name, value=False):del args[i:i + 1 + int(value)]+READER_DIR = ROOT / "engine-sycl" # the folder setup treats as the engine's: BUILD.json and, for pictures, the CPU reader strata-vision+++def intel_vision(asked, vram_gb: float = 0.0) -> str:+ """The picture reader wanted on an Intel Arc: "none" when switched off, else "cpu" (the only reader the SYCL port has; a+ SYCL GPU reader is later work). Takes setup.hip_vision's arguments."""+ return "none" if str(asked or "").lower() in ("no", "none", "off") else "cpu"++SYCL_PROFILE = "expert-profile-sycl.bin" # data/: the xe cards' expert ranking (to_sycl)SMALL_CARD_GB = 12.0 # under this the engine's own advice is --vram-reserve-mib 300 (a 1024 MiB reserve leaves an# 8 GB card no room for even one expert slot) and the English draft vocabulary@@ -222,7 +232,7 @@ def to_sycl(cfg: dict, exe: Path, ram: float, keep: dict, vram_gb: float = 0.0,if flag(args, "--prefill") in (None, "auto", "2048", "4096", "8192"):drop(args, "--prefill", True)args += ["--prefill", str(recommended_prefill(vram_gb, ctx, driver, devid)[0])]- out = {k: v for k, v in cfg.items() if k not in ("lib_dirs", "env", "vision", "gpus")}+ out = {k: v for k, v in cfg.items() if k not in ("lib_dirs", "env", "gpus")} # "vision" stays: the CPU reader runs on the hostout.update({"backend": "sycl", "exe": str(SYCL_WRAPPER), "args": args, "sycl_root": str(MOUNT)})env = {}if alchemist:@@ -264,9 +274,13 @@ def install(argv) -> None:keep[p.name] = {k: c[k] for k in ("model_switcher", "sampling") if k in c}except (OSError, ValueError):pass- stub = Path(tempfile.mkdtemp(prefix="strata-sycl-")) # setup reads the engine's BUILD.json; the SYCL build has none- atexit.register(shutil.rmtree, stub, True)- (stub / "BUILD.json").write_text(json.dumps({"version": sycl_version(), "source": "local", "lib_dirs": []}))+ stub = READER_DIR # setup reads the engine's BUILD.json; the SYCL build has none. A folder that+ stub.mkdir(exist_ok=True) # stays: the picture reader in it is what the run config points at+ try:+ stamped = json.loads((stub / "BUILD.json").read_text(encoding="utf-8")) # keeps what the reader's build recorded+ except (OSError, ValueError):+ stamped = {}+ (stub / "BUILD.json").write_text(json.dumps({**stamped, "version": sycl_version(), "source": "local", "lib_dirs": []}))say = S.sayfake_ram = max(real_ram, 1024.0)@@ -294,8 +308,15 @@ def install(argv) -> None:S.gpus = lambda *a, **k: []S.amd_gpus = lambda *a, **k: intelS.amd_problem = lambda g: None- S.hip_vision = lambda asked, vram_gb=0.0: "none" # images are not wired on the SYCL port yet- S.build_engine_hip = lambda *a, **k: stub+ S.hip_vision = intel_vision # pictures on by default, read on the CPU (setup's hip_reader settles it)++ def build_engine_hip(gpu, llama, vision="none"):+ """No engine to compile here (the SYCL build is separate); the picture reader is: the portable CPU reader, once."""+ if vision != "none":+ stamp = stub / "BUILD.json"+ S.build_vision_cpu(stub, stamp, json.loads(stamp.read_text(encoding="utf-8")), llama, S.source_hash(S.VISION_SOURCES))+ return stub+ S.build_engine_hip = build_engine_hipS.hipblaslt_table = lambda *a, **k: NoneS.ram_gb = lambda: fake_ram # the experts are in VRAM: setup's RAM rule does not apply@@ -376,8 +397,6 @@ def install(argv) -> None:sys.argv = [str(ROOT / "setup.py"), *argv]if "--backend" not in argv:sys.argv += ["--backend", "hip"]- if "--vision" not in argv:- sys.argv += ["--vision", "none"]if "--low-ram" not in argv:sys.argv += ["--low-ram", "off"]sys.exit(S.main())diff --git a/tools/test_setup_sycl.py b/tools/test_setup_sycl.pyindex 1c993a6ae..2f3ad0759 100644--- a/tools/test_setup_sycl.py+++ b/tools/test_setup_sycl.py@@ -223,6 +223,53 @@ class ToSycl(unittest.TestCase):self.assertTrue(m.small_card(10.0, "xe"))self.assertFalse(m.small_card(32.0, "xe"))+ def test_pictures_are_on_by_default_read_on_the_cpu_and_the_vision_section_survives(self):+ """Intel Arc: the CPU reader (strata-vision) on the host, --vision no is the off switch; "gpu" is read as "cpu" (no SYCL GPU reader)."""+ m = self.mod()+ for asked in (None, "yes", "gpu", "cpu", "on"):+ self.assertEqual(m.intel_vision(asked, 8.0), "cpu")+ self.assertEqual(m.intel_vision(asked, 32.0), "cpu")+ for asked in ("no", "none", "off"):+ self.assertEqual(m.intel_vision(asked, 32.0), "none")+ section = {"exe": "/h/engine-sycl/strata-vision", "mmproj": "/h/models/mmproj.gguf", "model": "/h/models/m.gguf", "gpu": False,+ "max_tokens": 300}+ cfg = self.cfg(m, ["--vision"])+ cfg["vision"] = section+ c = m.to_sycl(cfg, m.ROOT / "build-sycl-aot" / "strata", 64.0, {}, 32.0, "xe")+ self.assertEqual(c["vision"], section) # host paths: the reader runs on the host, not in the container+ self.assertIn("--vision", c["args"])++ def test_setup_intel_no_longer_forces_pictures_off(self):+ src = (ROOT / "sycl" / "setup_intel.py").read_text(encoding="utf-8")+ self.assertNotIn('sys.argv += ["--vision", "none"]', src)+ self.assertIn("S.hip_vision = intel_vision", src)+ self.assertIn("S.build_vision_cpu(", src) # the CPU reader is compiled by setup's own step++ def test_the_container_shares_dev_shm_with_the_server_for_the_embeddings(self):+ sh = (ROOT / "sycl" / "serve" / "strata-sycl.sh").read_text(encoding="utf-8")+ self.assertIn("-v /dev/shm:/dev/shm", sh)+ self.assertIn("STRATA_SYCL_SHM", sh)++ def test_a_picture_that_cannot_reach_the_container_is_said_once(self):+ import importlib.util+ spec = importlib.util.spec_from_file_location("server_intel_t", ROOT / "sycl" / "serve" / "server_intel.py")+ src = (ROOT / "sycl" / "serve" / "server_intel.py").read_text(encoding="utf-8")+ self.assertIn("install_shared_embeddings()", src.split('if __name__ == "__main__":')[1])+ sys.path.insert(0, str(ROOT / "sycl" / "serve"))+ try:+ m = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(m)+ except ImportError as e: # no oneAPI / xe telemetry import here: the source check above stands+ self.skipTest(str(e))+ finally:+ sys.path.pop(0)+ out = io.StringIO()+ with mock.patch.object(m.S, "combined_embeddings_path", lambda d, n: Path("/tmp/x/req.sve")), contextlib.redirect_stdout(out):+ m.install_shared_embeddings()+ m.S.combined_embeddings_path(Path("/tmp/x"), 10)+ m.S.combined_embeddings_path(Path("/tmp/x"), 10)+ self.assertEqual(out.getvalue().count("/dev/shm has no room"), 1)+if __name__ == "__main__":unittest.main()