Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dflash2: reserve the drafter's head before the expert cache is sized, loud fallback lines naming the VRAM needed, Blackwell window cap 6 (draft_cap_test)
Niko1221eebb870922b7
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 77671a8b4..009d526fb 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -1476,6 +1476,10 @@ if(STRATA_BUILD_TESTS OR STRATA_BUILD_CONVERSATION_TESTS)add_executable(spec_prob_test src/core/spec_prob_test.cpp)target_include_directories(spec_prob_test PRIVATE include)add_test(NAME spec_prob_test COMMAND spec_prob_test)+ # the per-card window cap of --draft dflash2 (header-only, CPU)+ add_executable(draft_cap_test src/core/draft_cap_test.cpp)+ target_include_directories(draft_cap_test PRIVATE include)+ add_test(NAME draft_cap_test COMMAND draft_cap_test)# the f32 -> bf16 rounding against ggml's, NaN included, over every f32 bit pattern (header-only, CPU)add_executable(bf16_bits_test src/kernels/bf16_bits_test.cpp)target_include_directories(bf16_bits_test PRIVATE include)diff --git a/include/strata/core/draft_cap.hpp b/include/strata/core/draft_cap.hppnew file mode 100644index 000000000..42d2038d6--- /dev/null+++ b/include/strata/core/draft_cap.hpp@@ -0,0 +1,21 @@+// include/strata/core/draft_cap.hpp - the widest verify window --draft dflash2 uses on each card class (measured A/Bs, notes-dflash2-p1.md; recommend, never force).+#pragma once++#include <cstddef>++namespace strata::core {++/// 0 = no cap (the --spec given stays). `uma`: Strix Halo / iGPU (gfx1151); `cc_major`: the CUDA compute capability's major (0 on HIP).+/// - Strix Halo / UMA: 5 (agent-safe; 6 ties on agent prompts)+/// - Blackwell (sm_120, cc 12): 6 (+1.9% at 6 on an RTX PRO 5000, 8 was worse)+/// - everything else: unmeasured, --spec as given+inline int dflash2_auto_cap(bool uma, int cc_major) {+ if (uma) return 5;+ if (cc_major == 12) return 6;+ return 0;+}++/// A 12 GB NVIDIA card (< 13 GiB, CUDA): the MTP drafter was faster in most measurements, which the engine says once.+inline bool dflash2_small_nvidia(bool cuda, size_t total_bytes) { return cuda && total_bytes < ((size_t) 13 << 30); }++} // namespace strata::corediff --git a/src/core/draft_cap_test.cpp b/src/core/draft_cap_test.cppnew file mode 100644index 000000000..7137a9239--- /dev/null+++ b/src/core/draft_cap_test.cpp@@ -0,0 +1,21 @@+// src/core/draft_cap_test.cpp - the per-card window cap of --draft dflash2 (CPU only).+#include "strata/core/draft_cap.hpp"++#include <cstdio>++using namespace strata::core;++static int fails = 0;+#define CHECK(c) do { if (!(c)) { std::fprintf(stderr, "FAIL %s:%d %s\n", __FILE__, __LINE__, #c); ++fails; } } while (0)++int main() {+ CHECK(dflash2_auto_cap(true, 0) == 5); // Strix Halo+ CHECK(dflash2_auto_cap(false, 12) == 6); // Blackwell (sm_120)+ CHECK(dflash2_auto_cap(false, 8) == 0); // Ampere / Ada: --spec as given+ CHECK(dflash2_auto_cap(false, 0) == 0); // another AMD card+ CHECK(dflash2_small_nvidia(true, (size_t) 12 << 30));+ CHECK(!dflash2_small_nvidia(true, (size_t) 16 << 30));+ CHECK(!dflash2_small_nvidia(false, (size_t) 8 << 30));+ if (fails == 0) std::printf("draft_cap_test: ok\n");+ return fails == 0 ? 0 : 1;+}diff --git a/src/program/generate.cpp b/src/program/generate.cppindex 9840d4f18..3ec650534 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -68,6 +68,7 @@#include "strata/core/mtp.hpp"#if defined(STRATA_HAVE_DFLASH2)#include "strata/core/dflash2.hpp"+#include "strata/core/draft_cap.hpp"#endif#include "strata/prefill/prefill.hpp"#if defined(STRATA_PREFILL_MMQ)@@ -5463,20 +5464,27 @@ int main(int argc, char** argv) {#elseconst bool uma = false;#endif- const int cap = o.draft_cap > 0 ? o.draft_cap : (uma ? 5 : 0);+#if defined(STRATA_USE_HIP)+ const int cc_major = 0;+#else+ const int cc_major = dp.major;+#endif+ const int cap = o.draft_cap > 0 ? o.draft_cap : strata::core::dflash2_auto_cap(uma, cc_major);if (cap > 0 && o.spec > cap) {std::fprintf(stderr, "strata generate: --draft dflash2: windows capped at %d tokens (%s; --draft-cap N changes it)\n", cap,o.draft_cap > 0 ? "as asked" : "the width measured best on this card");o.spec = cap;}#if !defined(STRATA_USE_HIP)- if (o.draft_cap == 0 && dp.totalGlobalMem < (size_t) 13 << 30)+ if (o.draft_cap == 0 && strata::core::dflash2_small_nvidia(true, dp.totalGlobalMem))std::fprintf(stderr, "strata generate: --draft dflash2: on a 12 GB NVIDIA card the MTP drafter was faster in most measurements ""(code prompts at a window of 6 gained 3%%); continuing as asked\n");#endif}} else {- std::fprintf(stderr, "strata generate: --draft dflash2: %s; the MTP drafts instead\n", err.c_str());+ std::fprintf(stderr, "strata generate: --draft dflash2: %s. DFlash2 needs about 1.2 GiB of VRAM (the pack 0.76 GiB, its buffers and K/V ring "+ "0.25 GiB, a draft head 0.2 GiB); start with --vram-reserve-mib 1900 to keep it (the expert cache then gives up about 2.5%% "+ "of its experts to the CPU), or a smaller --max-context. The MTP drafts instead.\n", err.c_str());}} else {std::fprintf(stderr, "strata generate: --draft dflash2 is off: %s; the MTP drafts instead\n", why);@@ -5826,6 +5834,9 @@ int main(int argc, char** argv) {const int64_t pf_mib = (o.prefill_chunk > 0 && !pf_borrow) ? 160 + (o.prefill_chunk * 680) / 1024 : 0;int64_t mtp_b = (!o.mtp.empty() && native_head.loaded())? (int64_t) mtp.bind_bytes(native_head.row_bytes(), n_vocab) : 0;+#if defined(STRATA_HAVE_DFLASH2)+ if (use_df2 && native_head.loaded()) mtp_b += (int64_t) mtp.bind_bytes(native_head.row_bytes(), n_vocab) + (48 << 20); // DFlash2's own draft head and logits+#endifif (stages.empty())for (const auto& d : slot_mtp) mtp_b += (int64_t) d->bind_bytes(native_head.row_bytes(), n_vocab);const int64_t pipe_b = o.pipeline_windows <= 0 ? 0@@ -6070,6 +6081,9 @@ int main(int argc, char** argv) {if (stages.empty())for (const auto& d : slot_mtp)mtp_bind += (int64_t) d->bind_bytes(native_head.row_bytes(), n_vocab);+#if defined(STRATA_HAVE_DFLASH2)+ if (use_df2 && native_head.loaded()) mtp_bind += (int64_t) mtp.bind_bytes(native_head.row_bytes(), n_vocab) + (48 << 20); // DFlash2's own draft head and logits, bound after this+#endifconst int64_t reserve = (((int64_t) o.vram_reserve_mib + prefill_mib) << 20) + mtp_bind + pipe_first;const int64_t blob = (int64_t) strata::kernels::cpu::expert_layout().max_blob;int64_t slots = ((int64_t) free_b - reserve) / blob;@@ -16052,11 +16066,15 @@ int main(int argc, char** argv) {use_df2 = false;}if (use_df2 && !df2.bind(wt, &native_head, o.mtp_draft_vocab.empty() ? o.mtp + "/draft_vocab.bin" : o.mtp_draft_vocab, err)) {- std::fprintf(stderr, "strata generate: --draft dflash2: %s; the MTP drafts instead\n", err.c_str());+ std::fprintf(stderr, "strata generate: --draft dflash2: %s. It needs about 1.2 GiB of VRAM in all (the pack 0.76 GiB, buffers and K/V ring "+ "0.25 GiB, a draft head 0.2 GiB); --vram-reserve-mib 1900 keeps it (the expert cache gives up about 2.5%% of its experts "+ "to the CPU). The MTP drafts instead.\n", err.c_str());use_df2 = false;}if (use_df2 && !df2.prepare(ver.tap_dev(), err)) {- std::fprintf(stderr, "strata generate: --draft dflash2: %s; the MTP drafts instead\n", err.c_str());+ std::fprintf(stderr, "strata generate: --draft dflash2: %s. It needs about 1.2 GiB of VRAM in all (the pack 0.76 GiB, buffers and K/V ring "+ "0.25 GiB, a draft head 0.2 GiB); --vram-reserve-mib 1900 keeps it (the expert cache gives up about 2.5%% of its experts "+ "to the CPU). The MTP drafts instead.\n", err.c_str());use_df2 = false;}if (use_df2) std::fprintf(stderr, "strata generate: DFlash2 drafter on (%s, graphs captured in %.0f ms), the MTP layer stays as the fallback\n", o.draft_model.c_str(), df2.ms_capture);