Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
[Engine] SYCL: the prompt attention batch is budgeted where it is allocated, and stays 32 by default
Niko1221870aedd9e1eb
diff --git a/sycl/src/prefill/prefill.cpp b/sycl/src/prefill/prefill.cppindex 8206fb614..4c0205fbc 100644--- a/sycl/src/prefill/prefill.cpp+++ b/sycl/src/prefill/prefill.cpp@@ -697,6 +697,22 @@ struct PeerPrefill {}};+// SYCL port: how many queries the prompt path hands the FP32 attention kernel at once (STRATA_ATTN_BATCH, default 32; the+// tensor-core kernel is opt-in and slower here). Its scratch is batch x chunks x n_head x 258 floats, about 0.41 GB at 32 and+// 1.65 GB at 128 on Flash-Next, independent of the prompt chunk, so the size that init allocates (attn_batch_for) and the size+// the budget counts (bytes_needed_impl, which sizes the expert cache and the prompt path's loan) must be the same number.+// 128 reads a prompt 1.1-1.3% faster on two Arc Pro B65s with the lane-cell score phase, but its extra 1.2 GB comes out of+// the expert cache on every card (an 8 GB A750 most), so the default stays 32.+static int64_t attn_batch_setting() {+ const char* v = std::getenv("STRATA_ATTN_BATCH");+ const long long n = v ? std::atoll(v) : 32;+ return (n >= 1 && n <= 1024) ? (int64_t) n : 32LL;+}+// A chunk of T tokens never runs more than T queries at once: the scratch of the rest would sit unused.+static int64_t attn_batch_for(size_t T) {+ return std::max<int64_t>(1, std::min<int64_t>(attn_batch_setting(), (int64_t) T));+}+struct Prefill::Impl {const core::WeightTable* wt = nullptr;const core::ModelGeometry* g = nullptr;@@ -734,11 +750,7 @@ struct Prefill::Impl {float* sel_scores = nullptr; // [sel_batch, max_blocks]int64_t sel_batch = 256, max_blocks = 0;float* attn_scratch = nullptr;- int64_t attn_batch = [] { // SYCL port: how many queries the prompt path hands the FP32 attention kernel at once- const char* v = std::getenv("STRATA_ATTN_BATCH"); // (the tensor-core kernel is opt-in and slower here)- const long long n = v ? std::atoll(v) : 128; // 32 was picked for a vector-matrix kernel; with the- return (n >= 1 && n <= 1024) ? (int64_t) n : 128LL; // lane-cell score phase 128 reads a 4,106-token prompt- }(), cap = 0; // 1.1% faster than 32 and a 16,380-token one 1.3%+ int64_t attn_batch = 32, cap = 0; // attn_batch: set by carve from the chunk (attn_batch_for)// MoEfloat *logits = nullptr, *w = nullptr, *GU = nullptr, *Dm = nullptr, *sgate = nullptr, *sup = nullptr,*shared = nullptr, *sg = nullptr;@@ -1445,6 +1457,7 @@ bool Prefill::carve(size_t T, void* alloc) {m.steps_dev = o.take<int32_t>(T * strata::kernels::kStepCount, ok);strata::kernels::QsaShapes s = core::qsa_shapes(g);m.cap = strata::kernels::qsa_selection_width(strata::kernels::kTopkMaxCells, s);+ m.attn_batch = attn_batch_for(T);m.max_blocks = ss.qsa_states[ss.qsa_primary()].max_cells / s.idx_block + 2;{// one region for the attention half's and the MoE half's scratch (see gdn_set_bytes)@@ -2166,7 +2179,7 @@ uint64_t Prefill::bytes_needed_impl(const core::ModelGeometry& g, const core::Sestrata::kernels::QsaShapes s = core::qsa_shapes(g);const int64_t cap = strata::kernels::qsa_selection_width(strata::kernels::kTopkMaxCells, s);const int64_t max_blocks = ss.qsa_states[ss.qsa_primary()].max_cells / s.idx_block + 2;- o.take<uint8_t>((size_t) std::max({gdn_set_bytes(T), qsa_set_bytes(T, cap, max_blocks, 256, 32, s),+ o.take<uint8_t>((size_t) std::max({gdn_set_bytes(T), qsa_set_bytes(T, cap, max_blocks, 256, attn_batch_for(T), s),moe_set_bytes(T, g.n_expert, fused_layout(T, src))}), ok);for (int i = 0; i < dq_slots(); ++i) { o.take<uint16_t>(1280 * 2560, ok); o.take<uint16_t>(2560 * 640, ok); }if (mmq_plan().any) {