Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

setup_intel: recommend --prefill 8192 for a Battlemage card in the borrowing (over 32K context) config, with the measured numbers and a test

Niko1221219ae436fa16
diff --git a/docs/INTEL.md b/docs/INTEL.md
index 41967e5ef..f987a5780 100644
--- a/docs/INTEL.md
+++ b/docs/INTEL.md
@@ -881,3 +881,21 @@ gate identical in the 32K and the 128K configuration); every one has an off swit
| decode | 23-25 tok/s; GPU 92% busy at 165 W, CPU idle |
| prefill | 149 tok/s on a 2,701-token prompt; 424 tok/s on a 104,798-token prompt at 131k context |
| quality | correct code on every test; matches the NVIDIA path token for token in spirit, not measured |
+
+## Arc Pro B70: `--prefill 8192` in the long-context config (2026-10-18, branch f-151-b70)
+
+`setup_intel.py` now recommends 8,192-token prompt chunks for a Battlemage card with 24 GB or more when the context is over 32K (the
+prompt borrows cache slots; `recommended_prefill()`), 4,096 everywhere else, and it keeps a chunk size you put in the config.
+Arc Pro B70, IQ3_S, setup's 128K flags (`--kv-resident 32768 --vram-reserve-mib 2048`), `--pcie-frac 0 --adapt-every 0`, 2 rounds,
+32 greedy tokens after the prompt (identical in every run):
+
+| prompt | `--prefill 4096` | `--prefill 8192` | cache slots | decode after |
+|---|---|---|---|---|
+| 16K | 1,222 tok/s | 1,347-1,362 (+10-11%) | 11,795 both | 33-38 / 38 |
+| 32K | 1,187 | 1,361-1,383 (+15-16%) | 11,794-11,795 both | 35 / 38.6 |
+
+The lent slots are refilled after the prompt, so the bigger chunk costs the decode nothing. First-token logits against 4096
+(`STRATA_DUMP_FIRST_LOGITS`, whole vocabulary): KL 1.1e-2 nats at 16K and 3.0e-2 at 32K, top-1 the same, largest logit
+difference 1.0 (the chunk boundary moves where the QSA block selection is made, so a different chunk size is not bit-identical;
+two runs of the same size are, KL 0). With the 32K context (own prompt buffers, no borrowing) a bigger chunk is +20% on a 16K
+prompt (1,236 -> 1,478-1,482) but takes 1,397 cache slots (10,926 -> 9,529) and decode after it falls 9-16%, so 4,096 stays there.
diff --git a/sycl/setup_intel.py b/sycl/setup_intel.py
index 1fd8bb871..2c1f8c2a8 100644
--- a/sycl/setup_intel.py
+++ b/sycl/setup_intel.py
@@ -145,6 +145,23 @@ def split_likely_short(arena_gb: float, vram_gbs: list) -> bool:
return arena_gb > 0 and total > 0 and arena_gb > SPLIT_SHORT_FRACTION * total
+PREFILL_BMG_MIN_GB = 24.0 # a Battlemage card with this much VRAM runs the prompt in 8,192-token chunks (to_sycl)
+
+
+def recommended_prefill(vram_gb: float, ctx: int, driver: str = "xe", devid: str = "") -> tuple:
+ """(chunk size, one plain line saying why or ""). 4096 everywhere (to_sycl's old rule), except a Battlemage card (xe
+ driver, not an A-series) with 24 GB or more and a context over 32K, where the prompt borrows cache slots: 8192.
+ Measured on the Arc Pro B70 (IQ3_S, 128K context, setup's flags, 2 interleaved rounds): a 16K prompt 1,222 -> 1,347-1,362
+ tok/s (+10-11%), a 32K prompt 1,187 -> 1,361-1,383 (+15-16%), the same 32 greedy tokens, the same 11,795 cache slots (the
+ lent slots are refilled after the prompt), decode after it unchanged (38 tok/s). With the 32K context (own prompt
+ buffers) a bigger chunk takes 1,400 cache slots from the decode (-9..-16% decode for +20% prompt), so that stays 4096."""
+ bmg = driver != "i915" and devid not in ALCHEMIST_IDS and devid in INTEL_ARC
+ if bmg and vram_gb >= PREFILL_BMG_MIN_GB and ctx > 32768:
+ return 8192, ("prompt reading: chunks of 8,192 tokens on this Arc (the prompt borrows cache slots, so decode keeps all "
+ "of them): 10-16% faster on 16K-32K prompts than 4,096 (change --prefill in the config to pick another)")
+ return 4096, ""
+
+
def to_sycl(cfg: dict, exe: Path, ram: float, keep: dict, vram_gb: float = 0.0, driver: str = "xe",
devid: str = "") -> dict:
"""setup's config (written for its HIP path) -> the SYCL port's: the container's paths, experts streamed from the
@@ -198,10 +215,13 @@ def to_sycl(cfg: dict, exe: Path, ram: float, keep: dict, vram_gb: float = 0.0,
else:
args += ["--stream-experts", "--vram-reserve-mib", reserve]
if ctx > 32768 or vram_gb >= 24: # long contexts: 4096-token chunks keep the prompt buffers small; a
- drop(args, "--prefill", True) # 24 GB+ card with part of the experts in the RAM mirror streams those
- args += ["--prefill", "4096"] # over PCIe once per chunk, so fewer, bigger chunks read the prompt
- # faster (Arc Pro B70, IQ3_S, a 4,095-token prompt: 2,048-token
- # chunks 618 tok/s, 4,096: 1,002; docs/INTEL.md); it costs ~700 cache slots
+ # 24 GB+ card with part of the experts in the RAM mirror streams those over PCIe once per chunk, so fewer, bigger
+ # chunks read the prompt faster (Arc Pro B70, IQ3_S, a 4,095-token prompt: 2,048-token chunks 618 tok/s, 4,096: 1,002;
+ # docs/INTEL.md); it costs ~700 cache slots. A Battlemage card with the prompt borrowing cache slots reads 8,192.
+ # A chunk size the user already put in the config is kept: this is a recommendation, not a rule.
+ if flag(args, "--prefill") in (None, "auto", "2048", "4096", "8192"):
+ drop(args, "--prefill", True)
+ args += ["--prefill", str(recommended_prefill(vram_gb, ctx, driver, devid)[0])]
out = {k: v for k, v in cfg.items() if k not in ("lib_dirs", "env", "vision", "gpus")}
out.update({"backend": "sycl", "exe": str(SYCL_WRAPPER), "args": args, "sycl_root": str(MOUNT)})
env = {}
@@ -325,6 +345,11 @@ def install(argv) -> None:
S.warn(f"{model} on this Arc computes the experts the card does not hold on the CPU (about {need} GB of "
f"weights; this PC has {real_ram:.0f} GB of RAM): the start will be slow or fail - a smaller model "
"(--model) fits better (docs/INTEL.md)")
+ if flag(cfg["args"], "--prefill") == "8192":
+ why = recommended_prefill(sel["vram_gb"], int(flag(cfg["args"], "--max-context") or 32768), sel["driver"],
+ sel.get("devid", ""))[1]
+ if why:
+ S.say(" " + why)
Path(cfg_path).write_text(json.dumps(cfg, indent=1), encoding="utf-8")
script = write(model, cfg_path, port, open_browser)
script.write_text(script.read_text().replace(str(ROOT / "serve" / "server.py"), str(SERVER)))
diff --git a/tests/sycl/test_setup_intel_prefill.py b/tests/sycl/test_setup_intel_prefill.py
new file mode 100644
index 000000000..0c7a742a2
--- /dev/null
+++ b/tests/sycl/test_setup_intel_prefill.py
@@ -0,0 +1,52 @@
+"""sycl/setup_intel.py: the prompt chunk size setup recommends for an Intel card (--prefill).
+
+A Battlemage card (xe driver, not an A-series) with 24 GB or more and a context over 32K (the prompt borrows cache slots) gets
+8192; everything else keeps 4096; a chunk size the user put in the config is kept."""
+import importlib.util
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+sys.path.insert(0, str(ROOT))
+_spec = importlib.util.spec_from_file_location("setup_intel", ROOT / "sycl" / "setup_intel.py")
+si = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(si)
+
+B70 = dict(vram_gb=32.0, driver="xe", devid="e223")
+A770 = dict(vram_gb=16.0, driver="xe", devid="56a0")
+A750 = dict(vram_gb=8.0, driver="i915", devid="56a1")
+
+
+def prefill_of(ctx, card, extra=()):
+ cfg = {"args": ["--max-context", str(ctx), "--kv", "int8", "--prefill", "auto", *extra]}
+ out = si.to_sycl(cfg, ROOT / "build-sycl-aot" / "strata", 64.0, {}, **card)
+ return si.flag(out["args"], "--prefill")
+
+
+def test_helper_values():
+ assert si.recommended_prefill(32.0, 131072, "xe", "e223")[0] == 8192
+ assert si.recommended_prefill(32.0, 131072, "xe", "e223")[1] # a plain line says why
+ assert si.recommended_prefill(24.0, 65536, "xe", "e221")[0] == 8192
+ assert si.recommended_prefill(32.0, 32768, "xe", "e223") == (4096, "") # own prompt buffers: a bigger chunk costs cache slots
+ assert si.recommended_prefill(16.0, 131072, "xe", "e212")[0] == 4096 # under 24 GB
+ assert si.recommended_prefill(16.0, 131072, "xe", "56a0")[0] == 4096 # Alchemist
+ assert si.recommended_prefill(8.0, 131072, "i915", "56a1")[0] == 4096
+
+
+def test_config_battlemage_long_context():
+ assert prefill_of(131072, B70) == "8192"
+
+
+def test_config_battlemage_32k_keeps_4096():
+ assert prefill_of(32768, B70) == "4096"
+
+
+def test_config_other_cards_unchanged():
+ assert prefill_of(131072, A770) == "4096"
+ assert prefill_of(131072, A750) == "4096"
+
+
+def test_user_pick_is_kept():
+ cfg = {"args": ["--max-context", "131072", "--kv", "int8", "--prefill", "16384"]}
+ out = si.to_sycl(cfg, ROOT / "build-sycl-aot" / "strata", 64.0, {}, **B70)
+ assert si.flag(out["args"], "--prefill") == "16384"