Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
[Engine] SYCL: --pipeline-windows 2 is the default on exactly two Arc Battlemage cards
Niko1221a6fd788567a9
diff --git a/sycl/CMakeLists.txt b/sycl/CMakeLists.txtindex 17e47c514..dac10518b 100644--- a/sycl/CMakeLists.txt+++ b/sycl/CMakeLists.txt@@ -441,6 +441,9 @@ if(STRATA_SYCL_PARITY)add_executable(esimd_moe_mode_test ${CMAKE_CURRENT_SOURCE_DIR}/src/kernels/esimd_moe_mode_test.cpp) # 151-b70: the default choice of the expert chain (no GPU)target_include_directories(esimd_moe_mode_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include)add_test(NAME esimd_moe_mode_test COMMAND esimd_moe_mode_test)+ add_executable(pipeline_auto_test ${CMAKE_CURRENT_SOURCE_DIR}/src/core/pipeline_auto_test.cpp) # the default of --pipeline-windows (no GPU)+ target_include_directories(pipeline_auto_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include)+ add_test(NAME pipeline_auto_test COMMAND pipeline_auto_test)if(_i8_esimd) # DPAS s8 8x16x32 (Xe2): not for an Alchemist / Xe-HPC AOT build, nor with STRATA_SYCL_I8_ESIMD=OFFadd_executable(esimd_dpas_probe ${CMAKE_CURRENT_SOURCE_DIR}/src/kernels/esimd_dpas_probe.cpp) # ESIMD DPAS peak (registers only)add_executable(esimd_i8_bench ${CMAKE_CURRENT_SOURCE_DIR}/src/kernels/esimd_i8_bench.cpp) # ESIMD int8 DPAS GEMM on pre-decoded weightsdiff --git a/sycl/include/strata/pipeline_auto.hpp b/sycl/include/strata/pipeline_auto.hppnew file mode 100644index 000000000..6198698b8--- /dev/null+++ b/sycl/include/strata/pipeline_auto.hpp@@ -0,0 +1,32 @@+// The default for --pipeline-windows when it is not given (host-only logic, no SYCL runtime: tested by+// src/core/pipeline_auto_test.cpp). On for a layer split of exactly two Arc Battlemage (Xe2) cards, with --serve and the+// MTP drafter; off for everything else. The caller finds `battlemage_*` from the devices' architecture ids+// (strata::intel_gpu_gen), never from a name.+#pragma once++namespace strata {++struct PipelineAutoInputs {+ bool serve = false; ///< --serve+ bool multi_gpu = false; ///< a layer split across GPUs+ bool split_same = false; ///< the split puts every stage on one GPU+ unsigned later_stages = 0; ///< GPUs after the first one: 1 = a split of exactly two stages+ bool auto_off = false; ///< STRATA_PIPELINE_AUTO=0+ bool has_mtp = false; ///< --mtp given+ int spec = 0; ///< --spec after the engine's own widening (>= 2 means a draft window)+ int batch = 0; ///< --batch slots+ bool peer_device = false; ///< --peer-device+ bool helper_caches = false; ///< --expert-cache-device1..3+ bool first_is_battlemage = false; ///< stage 0's GPU is a Battlemage card (architecture id)+ bool later_is_battlemage = false; ///< the second stage's GPU is a Battlemage card+};++/// True when --pipeline-windows 2 is the default for this run. Single card, an Arc A-series card (Alchemist), any other+/// vendor, three or more stages (the SYCL pipeline runs exactly two), --batch, --peer-device and the helper caches stay serial.+inline bool pipeline_auto_on(const PipelineAutoInputs& in) {+ return in.serve && in.multi_gpu && !in.split_same && in.later_stages == 1 && !in.auto_off && in.has_mtp &&+ in.spec >= 2 && in.batch == 0 && !in.peer_device && !in.helper_caches && in.first_is_battlemage &&+ in.later_is_battlemage;+}++} // namespace stratadiff --git a/sycl/src/core/pipeline_auto_test.cpp b/sycl/src/core/pipeline_auto_test.cppnew file mode 100644index 000000000..88ace2bb2--- /dev/null+++ b/sycl/src/core/pipeline_auto_test.cpp@@ -0,0 +1,40 @@+// The default of --pipeline-windows (strata::pipeline_auto_on), no GPU needed: on only for a layer split of exactly two+// Battlemage cards with --serve and the MTP drafter.+#include <cstdio>+#include "strata/pipeline_auto.hpp"++using strata::PipelineAutoInputs;+using strata::pipeline_auto_on;+static int bad = 0;+#define CHECK(cond, what) do { if (!(cond)) { std::printf("FAIL: %s\n", what); ++bad; } } while (0)++static PipelineAutoInputs two_b65() {+ PipelineAutoInputs in;+ in.serve = true; in.multi_gpu = true; in.later_stages = 1; in.has_mtp = true; in.spec = 4;+ in.first_is_battlemage = true; in.later_is_battlemage = true;+ return in;+}++int main() {+ CHECK(pipeline_auto_on(two_b65()), "two Battlemage cards, serve, MTP -> on");+ { auto in = two_b65(); in.spec = 2; CHECK(pipeline_auto_on(in), "spec 2 -> on"); }+ { auto in = two_b65(); in.spec = 1; CHECK(!pipeline_auto_on(in), "spec 1 (no draft window) -> off"); }+ { auto in = two_b65(); in.spec = 0; CHECK(!pipeline_auto_on(in), "no --spec -> off"); }+ { auto in = two_b65(); in.has_mtp = false; CHECK(!pipeline_auto_on(in), "no MTP drafter -> off"); }+ { auto in = two_b65(); in.serve = false; CHECK(!pipeline_auto_on(in), "no --serve -> off"); }+ { auto in = two_b65(); in.multi_gpu = false; CHECK(!pipeline_auto_on(in), "single card -> off"); }+ { auto in = two_b65(); in.split_same = true; CHECK(!pipeline_auto_on(in), "both stages on one GPU -> off"); }+ { auto in = two_b65(); in.later_stages = 0; CHECK(!pipeline_auto_on(in), "no later stage -> off"); }+ { auto in = two_b65(); in.later_stages = 2; CHECK(!pipeline_auto_on(in), "three stages (the pipeline runs exactly two) -> off"); }+ { auto in = two_b65(); in.later_stages = 3; CHECK(!pipeline_auto_on(in), "four stages -> off"); }+ { auto in = two_b65(); in.first_is_battlemage = false; CHECK(!pipeline_auto_on(in), "first card not Battlemage (A-series) -> off"); }+ { auto in = two_b65(); in.later_is_battlemage = false; CHECK(!pipeline_auto_on(in), "second card not Battlemage -> off"); }+ { auto in = two_b65(); in.first_is_battlemage = false; in.later_is_battlemage = false;+ CHECK(!pipeline_auto_on(in), "two A-series cards -> off"); }+ { auto in = two_b65(); in.auto_off = true; CHECK(!pipeline_auto_on(in), "STRATA_PIPELINE_AUTO=0 -> off"); }+ { auto in = two_b65(); in.batch = 4; CHECK(!pipeline_auto_on(in), "--batch slots -> off"); }+ { auto in = two_b65(); in.peer_device = true; CHECK(!pipeline_auto_on(in), "--peer-device -> off"); }+ { auto in = two_b65(); in.helper_caches = true; CHECK(!pipeline_auto_on(in), "helper caches -> off"); }+ if (bad == 0) std::printf("pipeline_auto_test: all OK\n");+ return bad == 0 ? 0 : 1;+}diff --git a/sycl/src/program/generate.cpp b/sycl/src/program/generate.cppindex 3db8caef9..1946ef5e1 100644--- a/sycl/src/program/generate.cpp+++ b/sycl/src/program/generate.cpp@@ -25,6 +25,7 @@#include <sycl/sycl.hpp>#include <dpct/dpct.hpp>#include "strata/sycl_queue.hpp"+#include "strata/pipeline_auto.hpp"#include "strata/sycl_doorbell.hpp"#include "strata/core/arch_defaults.hpp"#include "strata/core/dma_batch.hpp"@@ -704,11 +705,12 @@ struct Options {/// result) goes through the verify windows, S tokens at a time, instead of the batched prompt path (0 = always/// the batched path)int64_t short_read = 64;- /// --pipeline-windows N (--serve, a layer split on exactly two GPUs; opt-in): 1 = a prompt's short reads through+ /// --pipeline-windows N (--serve, a layer split on exactly two GPUs): 1 = a prompt's short reads through/// the verify windows run with the stages overlapped (stage 0 reads window K+1 while stage 1 reads window K);/// 2 = the decode windows too: stage 0 runs window K+1 speculatively while stage 1 verifies window K (rolled back- /// when K is not accepted as guessed). 0 = off (the stages take turns).- int pipeline_windows = 0;+ /// when K is not accepted as guessed). 0 = off (the stages take turns). -1 (not given) = automatic: 2 when both+ /// GPUs of the two-stage split are Battlemage (Xe2) cards and the MTP drafter runs (spec >= 2), else 0.+ int pipeline_windows = -1;/// The suffix drafter (prompt lookup): when the text being written repeats an earlier stretch of the context (code/// edits, quoted input, tool-call JSON) by at least this many tokens, the window may be filled with what followed/// it there instead of the MTP's drafts, where the MTP's own first guess agrees and the draft policy expects it to@@ -834,7 +836,8 @@ void usage() {" --batch-mtp --batch (opt-in, one GPU, needs --mtp and --spec): each slot also verifies one MTP\n"" proposal per window (STRATA_BATCH_MTP=1 does the same); needs VRAM per slot\n"" --batch-groups G --batch with a layer split: the slots in G groups pipelined through the GPUs\n"- " --pipeline-windows N --serve with a layer split on two GPUs (opt-in): one conversation's verify windows\n"+ " --pipeline-windows N --serve with a layer split on two GPUs (default 2 on two Arc B-series cards with\n"+ " the MTP drafter, else 0): one conversation's verify windows\n"" with the GPUs overlapped - 1 = the prompt's short reads, 2 = decode as well\n"" (stage 0 runs the next window while stage 1 verifies this one); docs/MULTI_GPU.md\n"" --trim-stage-weights an explicit --layer-split: each GPU loads only its own layers' dense weights\n"@@ -3387,9 +3390,39 @@ int main(int argc, char **argv) try {}}strata::core::Verifier::set_commit_async(!multi_gpu); // see Verifier::set_commit_async- // --pipeline-windows (opt-in): one conversation's windows with the two stages of a layer split overlapped. Decided+ // --pipeline-windows: one conversation's windows with the two stages of a layer split overlapped. Decided// here, before any stage sizes its expert cache (the second verifier per stage and the snapshots are allocated// after the caches, so their room is kept out of them). What it does not support turns it off, said once.+ // Not given: on by default (2) for a split of exactly two Battlemage (Xe2) cards, found from the devices'+ // architecture ids (strata::intel_gpu_gen), with --serve and the MTP drafter; a single card, an Arc A-series card or+ // anything the pipeline does not support stays serial, said nothing. STRATA_PIPELINE_AUTO=0 or+ // --pipeline-windows 0 turns it off.+ if (o.pipeline_windows < 0) {+ static const char* ae = std::getenv("STRATA_PIPELINE_AUTO");+ const auto battlemage = [](int d) -> bool {+ try {+ return strata::intel_gpu_gen(dpct::get_device(d)) == strata::IntelGpuGen::Battlemage;+ } catch (...) {+ return false;+ }+ };+ strata::PipelineAutoInputs in;+ in.serve = o.serve; in.multi_gpu = multi_gpu; in.split_same = split_same;+ in.later_stages = (unsigned) split_devs.size();+ in.auto_off = ae != nullptr && std::atoi(ae) == 0;+ in.has_mtp = !o.mtp.empty(); in.spec = o.spec; in.batch = o.batch;+ in.peer_device = o.peer_device >= 1; // the same test as the validation below+ in.helper_caches = o.expert_cache_remote[0] > 0 || o.expert_cache_remote[1] > 0 || o.expert_cache_remote[2] > 0;+ if (split_devs.size() == 1) { // the devices are looked at only for a split the pipeline could take+ in.first_is_battlemage = battlemage(0);+ in.later_is_battlemage = battlemage(split_devs[0]);+ }+ const bool on = strata::pipeline_auto_on(in);+ o.pipeline_windows = on ? 2 : 0;+ if (on)+ std::fprintf(stderr, "strata generate: --pipeline-windows 2 is on by default for a layer split across two Arc "+ "B-series cards (--pipeline-windows 0 or STRATA_PIPELINE_AUTO=0 turns it off)\n");+ }if (o.pipeline_windows > 0) {const bool helpers = o.expert_cache_remote[0] > 0 || o.expert_cache_remote[1] > 0 || o.expert_cache_remote[2] > 0;const char* off = !o.serve ? "it needs --serve"