Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

[Engine] SYCL: --pipeline-windows 2 is the default on exactly two Arc Battlemage cards

Niko1221a6fd788567a9
diff --git a/sycl/CMakeLists.txt b/sycl/CMakeLists.txt
index 17e47c514..dac10518b 100644
--- a/sycl/CMakeLists.txt
+++ b/sycl/CMakeLists.txt
@@ -441,6 +441,9 @@ if(STRATA_SYCL_PARITY)
add_executable(esimd_moe_mode_test ${CMAKE_CURRENT_SOURCE_DIR}/src/kernels/esimd_moe_mode_test.cpp) # 151-b70: the default choice of the expert chain (no GPU)
target_include_directories(esimd_moe_mode_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include)
add_test(NAME esimd_moe_mode_test COMMAND esimd_moe_mode_test)
+ add_executable(pipeline_auto_test ${CMAKE_CURRENT_SOURCE_DIR}/src/core/pipeline_auto_test.cpp) # the default of --pipeline-windows (no GPU)
+ target_include_directories(pipeline_auto_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include)
+ add_test(NAME pipeline_auto_test COMMAND pipeline_auto_test)
if(_i8_esimd) # DPAS s8 8x16x32 (Xe2): not for an Alchemist / Xe-HPC AOT build, nor with STRATA_SYCL_I8_ESIMD=OFF
add_executable(esimd_dpas_probe ${CMAKE_CURRENT_SOURCE_DIR}/src/kernels/esimd_dpas_probe.cpp) # ESIMD DPAS peak (registers only)
add_executable(esimd_i8_bench ${CMAKE_CURRENT_SOURCE_DIR}/src/kernels/esimd_i8_bench.cpp) # ESIMD int8 DPAS GEMM on pre-decoded weights
diff --git a/sycl/include/strata/pipeline_auto.hpp b/sycl/include/strata/pipeline_auto.hpp
new file mode 100644
index 000000000..6198698b8
--- /dev/null
+++ b/sycl/include/strata/pipeline_auto.hpp
@@ -0,0 +1,32 @@
+// The default for --pipeline-windows when it is not given (host-only logic, no SYCL runtime: tested by
+// src/core/pipeline_auto_test.cpp). On for a layer split of exactly two Arc Battlemage (Xe2) cards, with --serve and the
+// MTP drafter; off for everything else. The caller finds `battlemage_*` from the devices' architecture ids
+// (strata::intel_gpu_gen), never from a name.
+#pragma once
+
+namespace strata {
+
+struct PipelineAutoInputs {
+ bool serve = false; ///< --serve
+ bool multi_gpu = false; ///< a layer split across GPUs
+ bool split_same = false; ///< the split puts every stage on one GPU
+ unsigned later_stages = 0; ///< GPUs after the first one: 1 = a split of exactly two stages
+ bool auto_off = false; ///< STRATA_PIPELINE_AUTO=0
+ bool has_mtp = false; ///< --mtp given
+ int spec = 0; ///< --spec after the engine's own widening (>= 2 means a draft window)
+ int batch = 0; ///< --batch slots
+ bool peer_device = false; ///< --peer-device
+ bool helper_caches = false; ///< --expert-cache-device1..3
+ bool first_is_battlemage = false; ///< stage 0's GPU is a Battlemage card (architecture id)
+ bool later_is_battlemage = false; ///< the second stage's GPU is a Battlemage card
+};
+
+/// True when --pipeline-windows 2 is the default for this run. Single card, an Arc A-series card (Alchemist), any other
+/// vendor, three or more stages (the SYCL pipeline runs exactly two), --batch, --peer-device and the helper caches stay serial.
+inline bool pipeline_auto_on(const PipelineAutoInputs& in) {
+ return in.serve && in.multi_gpu && !in.split_same && in.later_stages == 1 && !in.auto_off && in.has_mtp &&
+ in.spec >= 2 && in.batch == 0 && !in.peer_device && !in.helper_caches && in.first_is_battlemage &&
+ in.later_is_battlemage;
+}
+
+} // namespace strata
diff --git a/sycl/src/core/pipeline_auto_test.cpp b/sycl/src/core/pipeline_auto_test.cpp
new file mode 100644
index 000000000..88ace2bb2
--- /dev/null
+++ b/sycl/src/core/pipeline_auto_test.cpp
@@ -0,0 +1,40 @@
+// The default of --pipeline-windows (strata::pipeline_auto_on), no GPU needed: on only for a layer split of exactly two
+// Battlemage cards with --serve and the MTP drafter.
+#include <cstdio>
+#include "strata/pipeline_auto.hpp"
+
+using strata::PipelineAutoInputs;
+using strata::pipeline_auto_on;
+static int bad = 0;
+#define CHECK(cond, what) do { if (!(cond)) { std::printf("FAIL: %s\n", what); ++bad; } } while (0)
+
+static PipelineAutoInputs two_b65() {
+ PipelineAutoInputs in;
+ in.serve = true; in.multi_gpu = true; in.later_stages = 1; in.has_mtp = true; in.spec = 4;
+ in.first_is_battlemage = true; in.later_is_battlemage = true;
+ return in;
+}
+
+int main() {
+ CHECK(pipeline_auto_on(two_b65()), "two Battlemage cards, serve, MTP -> on");
+ { auto in = two_b65(); in.spec = 2; CHECK(pipeline_auto_on(in), "spec 2 -> on"); }
+ { auto in = two_b65(); in.spec = 1; CHECK(!pipeline_auto_on(in), "spec 1 (no draft window) -> off"); }
+ { auto in = two_b65(); in.spec = 0; CHECK(!pipeline_auto_on(in), "no --spec -> off"); }
+ { auto in = two_b65(); in.has_mtp = false; CHECK(!pipeline_auto_on(in), "no MTP drafter -> off"); }
+ { auto in = two_b65(); in.serve = false; CHECK(!pipeline_auto_on(in), "no --serve -> off"); }
+ { auto in = two_b65(); in.multi_gpu = false; CHECK(!pipeline_auto_on(in), "single card -> off"); }
+ { auto in = two_b65(); in.split_same = true; CHECK(!pipeline_auto_on(in), "both stages on one GPU -> off"); }
+ { auto in = two_b65(); in.later_stages = 0; CHECK(!pipeline_auto_on(in), "no later stage -> off"); }
+ { auto in = two_b65(); in.later_stages = 2; CHECK(!pipeline_auto_on(in), "three stages (the pipeline runs exactly two) -> off"); }
+ { auto in = two_b65(); in.later_stages = 3; CHECK(!pipeline_auto_on(in), "four stages -> off"); }
+ { auto in = two_b65(); in.first_is_battlemage = false; CHECK(!pipeline_auto_on(in), "first card not Battlemage (A-series) -> off"); }
+ { auto in = two_b65(); in.later_is_battlemage = false; CHECK(!pipeline_auto_on(in), "second card not Battlemage -> off"); }
+ { auto in = two_b65(); in.first_is_battlemage = false; in.later_is_battlemage = false;
+ CHECK(!pipeline_auto_on(in), "two A-series cards -> off"); }
+ { auto in = two_b65(); in.auto_off = true; CHECK(!pipeline_auto_on(in), "STRATA_PIPELINE_AUTO=0 -> off"); }
+ { auto in = two_b65(); in.batch = 4; CHECK(!pipeline_auto_on(in), "--batch slots -> off"); }
+ { auto in = two_b65(); in.peer_device = true; CHECK(!pipeline_auto_on(in), "--peer-device -> off"); }
+ { auto in = two_b65(); in.helper_caches = true; CHECK(!pipeline_auto_on(in), "helper caches -> off"); }
+ if (bad == 0) std::printf("pipeline_auto_test: all OK\n");
+ return bad == 0 ? 0 : 1;
+}
diff --git a/sycl/src/program/generate.cpp b/sycl/src/program/generate.cpp
index 3db8caef9..1946ef5e1 100644
--- a/sycl/src/program/generate.cpp
+++ b/sycl/src/program/generate.cpp
@@ -25,6 +25,7 @@
#include <sycl/sycl.hpp>
#include <dpct/dpct.hpp>
#include "strata/sycl_queue.hpp"
+#include "strata/pipeline_auto.hpp"
#include "strata/sycl_doorbell.hpp"
#include "strata/core/arch_defaults.hpp"
#include "strata/core/dma_batch.hpp"
@@ -704,11 +705,12 @@ struct Options {
/// result) goes through the verify windows, S tokens at a time, instead of the batched prompt path (0 = always
/// the batched path)
int64_t short_read = 64;
- /// --pipeline-windows N (--serve, a layer split on exactly two GPUs; opt-in): 1 = a prompt's short reads through
+ /// --pipeline-windows N (--serve, a layer split on exactly two GPUs): 1 = a prompt's short reads through
/// the verify windows run with the stages overlapped (stage 0 reads window K+1 while stage 1 reads window K);
/// 2 = the decode windows too: stage 0 runs window K+1 speculatively while stage 1 verifies window K (rolled back
- /// when K is not accepted as guessed). 0 = off (the stages take turns).
- int pipeline_windows = 0;
+ /// when K is not accepted as guessed). 0 = off (the stages take turns). -1 (not given) = automatic: 2 when both
+ /// GPUs of the two-stage split are Battlemage (Xe2) cards and the MTP drafter runs (spec >= 2), else 0.
+ int pipeline_windows = -1;
/// The suffix drafter (prompt lookup): when the text being written repeats an earlier stretch of the context (code
/// edits, quoted input, tool-call JSON) by at least this many tokens, the window may be filled with what followed
/// it there instead of the MTP's drafts, where the MTP's own first guess agrees and the draft policy expects it to
@@ -834,7 +836,8 @@ void usage() {
" --batch-mtp --batch (opt-in, one GPU, needs --mtp and --spec): each slot also verifies one MTP\n"
" proposal per window (STRATA_BATCH_MTP=1 does the same); needs VRAM per slot\n"
" --batch-groups G --batch with a layer split: the slots in G groups pipelined through the GPUs\n"
- " --pipeline-windows N --serve with a layer split on two GPUs (opt-in): one conversation's verify windows\n"
+ " --pipeline-windows N --serve with a layer split on two GPUs (default 2 on two Arc B-series cards with\n"
+ " the MTP drafter, else 0): one conversation's verify windows\n"
" with the GPUs overlapped - 1 = the prompt's short reads, 2 = decode as well\n"
" (stage 0 runs the next window while stage 1 verifies this one); docs/MULTI_GPU.md\n"
" --trim-stage-weights an explicit --layer-split: each GPU loads only its own layers' dense weights\n"
@@ -3387,9 +3390,39 @@ int main(int argc, char **argv) try {
}
}
strata::core::Verifier::set_commit_async(!multi_gpu); // see Verifier::set_commit_async
- // --pipeline-windows (opt-in): one conversation's windows with the two stages of a layer split overlapped. Decided
+ // --pipeline-windows: one conversation's windows with the two stages of a layer split overlapped. Decided
// here, before any stage sizes its expert cache (the second verifier per stage and the snapshots are allocated
// after the caches, so their room is kept out of them). What it does not support turns it off, said once.
+ // Not given: on by default (2) for a split of exactly two Battlemage (Xe2) cards, found from the devices'
+ // architecture ids (strata::intel_gpu_gen), with --serve and the MTP drafter; a single card, an Arc A-series card or
+ // anything the pipeline does not support stays serial, said nothing. STRATA_PIPELINE_AUTO=0 or
+ // --pipeline-windows 0 turns it off.
+ if (o.pipeline_windows < 0) {
+ static const char* ae = std::getenv("STRATA_PIPELINE_AUTO");
+ const auto battlemage = [](int d) -> bool {
+ try {
+ return strata::intel_gpu_gen(dpct::get_device(d)) == strata::IntelGpuGen::Battlemage;
+ } catch (...) {
+ return false;
+ }
+ };
+ strata::PipelineAutoInputs in;
+ in.serve = o.serve; in.multi_gpu = multi_gpu; in.split_same = split_same;
+ in.later_stages = (unsigned) split_devs.size();
+ in.auto_off = ae != nullptr && std::atoi(ae) == 0;
+ in.has_mtp = !o.mtp.empty(); in.spec = o.spec; in.batch = o.batch;
+ in.peer_device = o.peer_device >= 1; // the same test as the validation below
+ in.helper_caches = o.expert_cache_remote[0] > 0 || o.expert_cache_remote[1] > 0 || o.expert_cache_remote[2] > 0;
+ if (split_devs.size() == 1) { // the devices are looked at only for a split the pipeline could take
+ in.first_is_battlemage = battlemage(0);
+ in.later_is_battlemage = battlemage(split_devs[0]);
+ }
+ const bool on = strata::pipeline_auto_on(in);
+ o.pipeline_windows = on ? 2 : 0;
+ if (on)
+ std::fprintf(stderr, "strata generate: --pipeline-windows 2 is on by default for a layer split across two Arc "
+ "B-series cards (--pipeline-windows 0 or STRATA_PIPELINE_AUTO=0 turns it off)\n");
+ }
if (o.pipeline_windows > 0) {
const bool helpers = o.expert_cache_remote[0] > 0 || o.expert_cache_remote[1] > 0 || o.expert_cache_remote[2] > 0;
const char* off = !o.serve ? "it needs --serve"