Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dflash2: reserve the drafter's head before the expert cache is sized, loud fallback lines naming the VRAM needed, Blackwell window cap 6 (draft_cap_test)

Niko1221eebb870922b7
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 77671a8b4..009d526fb 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1476,6 +1476,10 @@ if(STRATA_BUILD_TESTS OR STRATA_BUILD_CONVERSATION_TESTS)
add_executable(spec_prob_test src/core/spec_prob_test.cpp)
target_include_directories(spec_prob_test PRIVATE include)
add_test(NAME spec_prob_test COMMAND spec_prob_test)
+ # the per-card window cap of --draft dflash2 (header-only, CPU)
+ add_executable(draft_cap_test src/core/draft_cap_test.cpp)
+ target_include_directories(draft_cap_test PRIVATE include)
+ add_test(NAME draft_cap_test COMMAND draft_cap_test)
# the f32 -> bf16 rounding against ggml's, NaN included, over every f32 bit pattern (header-only, CPU)
add_executable(bf16_bits_test src/kernels/bf16_bits_test.cpp)
target_include_directories(bf16_bits_test PRIVATE include)
diff --git a/include/strata/core/draft_cap.hpp b/include/strata/core/draft_cap.hpp
new file mode 100644
index 000000000..42d2038d6
--- /dev/null
+++ b/include/strata/core/draft_cap.hpp
@@ -0,0 +1,21 @@
+// include/strata/core/draft_cap.hpp - the widest verify window --draft dflash2 uses on each card class (measured A/Bs, notes-dflash2-p1.md; recommend, never force).
+#pragma once
+
+#include <cstddef>
+
+namespace strata::core {
+
+/// 0 = no cap (the --spec given stays). `uma`: Strix Halo / iGPU (gfx1151); `cc_major`: the CUDA compute capability's major (0 on HIP).
+/// - Strix Halo / UMA: 5 (agent-safe; 6 ties on agent prompts)
+/// - Blackwell (sm_120, cc 12): 6 (+1.9% at 6 on an RTX PRO 5000, 8 was worse)
+/// - everything else: unmeasured, --spec as given
+inline int dflash2_auto_cap(bool uma, int cc_major) {
+ if (uma) return 5;
+ if (cc_major == 12) return 6;
+ return 0;
+}
+
+/// A 12 GB NVIDIA card (< 13 GiB, CUDA): the MTP drafter was faster in most measurements, which the engine says once.
+inline bool dflash2_small_nvidia(bool cuda, size_t total_bytes) { return cuda && total_bytes < ((size_t) 13 << 30); }
+
+} // namespace strata::core
diff --git a/src/core/draft_cap_test.cpp b/src/core/draft_cap_test.cpp
new file mode 100644
index 000000000..7137a9239
--- /dev/null
+++ b/src/core/draft_cap_test.cpp
@@ -0,0 +1,21 @@
+// src/core/draft_cap_test.cpp - the per-card window cap of --draft dflash2 (CPU only).
+#include "strata/core/draft_cap.hpp"
+
+#include <cstdio>
+
+using namespace strata::core;
+
+static int fails = 0;
+#define CHECK(c) do { if (!(c)) { std::fprintf(stderr, "FAIL %s:%d %s\n", __FILE__, __LINE__, #c); ++fails; } } while (0)
+
+int main() {
+ CHECK(dflash2_auto_cap(true, 0) == 5); // Strix Halo
+ CHECK(dflash2_auto_cap(false, 12) == 6); // Blackwell (sm_120)
+ CHECK(dflash2_auto_cap(false, 8) == 0); // Ampere / Ada: --spec as given
+ CHECK(dflash2_auto_cap(false, 0) == 0); // another AMD card
+ CHECK(dflash2_small_nvidia(true, (size_t) 12 << 30));
+ CHECK(!dflash2_small_nvidia(true, (size_t) 16 << 30));
+ CHECK(!dflash2_small_nvidia(false, (size_t) 8 << 30));
+ if (fails == 0) std::printf("draft_cap_test: ok\n");
+ return fails == 0 ? 0 : 1;
+}
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index 9840d4f18..3ec650534 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -68,6 +68,7 @@
#include "strata/core/mtp.hpp"
#if defined(STRATA_HAVE_DFLASH2)
#include "strata/core/dflash2.hpp"
+#include "strata/core/draft_cap.hpp"
#endif
#include "strata/prefill/prefill.hpp"
#if defined(STRATA_PREFILL_MMQ)
@@ -5463,20 +5464,27 @@ int main(int argc, char** argv) {
#else
const bool uma = false;
#endif
- const int cap = o.draft_cap > 0 ? o.draft_cap : (uma ? 5 : 0);
+#if defined(STRATA_USE_HIP)
+ const int cc_major = 0;
+#else
+ const int cc_major = dp.major;
+#endif
+ const int cap = o.draft_cap > 0 ? o.draft_cap : strata::core::dflash2_auto_cap(uma, cc_major);
if (cap > 0 && o.spec > cap) {
std::fprintf(stderr, "strata generate: --draft dflash2: windows capped at %d tokens (%s; --draft-cap N changes it)\n", cap,
o.draft_cap > 0 ? "as asked" : "the width measured best on this card");
o.spec = cap;
}
#if !defined(STRATA_USE_HIP)
- if (o.draft_cap == 0 && dp.totalGlobalMem < (size_t) 13 << 30)
+ if (o.draft_cap == 0 && strata::core::dflash2_small_nvidia(true, dp.totalGlobalMem))
std::fprintf(stderr, "strata generate: --draft dflash2: on a 12 GB NVIDIA card the MTP drafter was faster in most measurements "
"(code prompts at a window of 6 gained 3%%); continuing as asked\n");
#endif
}
} else {
- std::fprintf(stderr, "strata generate: --draft dflash2: %s; the MTP drafts instead\n", err.c_str());
+ std::fprintf(stderr, "strata generate: --draft dflash2: %s. DFlash2 needs about 1.2 GiB of VRAM (the pack 0.76 GiB, its buffers and K/V ring "
+ "0.25 GiB, a draft head 0.2 GiB); start with --vram-reserve-mib 1900 to keep it (the expert cache then gives up about 2.5%% "
+ "of its experts to the CPU), or a smaller --max-context. The MTP drafts instead.\n", err.c_str());
}
} else {
std::fprintf(stderr, "strata generate: --draft dflash2 is off: %s; the MTP drafts instead\n", why);
@@ -5826,6 +5834,9 @@ int main(int argc, char** argv) {
const int64_t pf_mib = (o.prefill_chunk > 0 && !pf_borrow) ? 160 + (o.prefill_chunk * 680) / 1024 : 0;
int64_t mtp_b = (!o.mtp.empty() && native_head.loaded())
? (int64_t) mtp.bind_bytes(native_head.row_bytes(), n_vocab) : 0;
+#if defined(STRATA_HAVE_DFLASH2)
+ if (use_df2 && native_head.loaded()) mtp_b += (int64_t) mtp.bind_bytes(native_head.row_bytes(), n_vocab) + (48 << 20); // DFlash2's own draft head and logits
+#endif
if (stages.empty())
for (const auto& d : slot_mtp) mtp_b += (int64_t) d->bind_bytes(native_head.row_bytes(), n_vocab);
const int64_t pipe_b = o.pipeline_windows <= 0 ? 0
@@ -6070,6 +6081,9 @@ int main(int argc, char** argv) {
if (stages.empty())
for (const auto& d : slot_mtp)
mtp_bind += (int64_t) d->bind_bytes(native_head.row_bytes(), n_vocab);
+#if defined(STRATA_HAVE_DFLASH2)
+ if (use_df2 && native_head.loaded()) mtp_bind += (int64_t) mtp.bind_bytes(native_head.row_bytes(), n_vocab) + (48 << 20); // DFlash2's own draft head and logits, bound after this
+#endif
const int64_t reserve = (((int64_t) o.vram_reserve_mib + prefill_mib) << 20) + mtp_bind + pipe_first;
const int64_t blob = (int64_t) strata::kernels::cpu::expert_layout().max_blob;
int64_t slots = ((int64_t) free_b - reserve) / blob;
@@ -16052,11 +16066,15 @@ int main(int argc, char** argv) {
use_df2 = false;
}
if (use_df2 && !df2.bind(wt, &native_head, o.mtp_draft_vocab.empty() ? o.mtp + "/draft_vocab.bin" : o.mtp_draft_vocab, err)) {
- std::fprintf(stderr, "strata generate: --draft dflash2: %s; the MTP drafts instead\n", err.c_str());
+ std::fprintf(stderr, "strata generate: --draft dflash2: %s. It needs about 1.2 GiB of VRAM in all (the pack 0.76 GiB, buffers and K/V ring "
+ "0.25 GiB, a draft head 0.2 GiB); --vram-reserve-mib 1900 keeps it (the expert cache gives up about 2.5%% of its experts "
+ "to the CPU). The MTP drafts instead.\n", err.c_str());
use_df2 = false;
}
if (use_df2 && !df2.prepare(ver.tap_dev(), err)) {
- std::fprintf(stderr, "strata generate: --draft dflash2: %s; the MTP drafts instead\n", err.c_str());
+ std::fprintf(stderr, "strata generate: --draft dflash2: %s. It needs about 1.2 GiB of VRAM in all (the pack 0.76 GiB, buffers and K/V ring "
+ "0.25 GiB, a draft head 0.2 GiB); --vram-reserve-mib 1900 keeps it (the expert cache gives up about 2.5%% of its experts "
+ "to the CPU). The MTP drafts instead.\n", err.c_str());
use_df2 = false;
}
if (use_df2) std::fprintf(stderr, "strata generate: DFlash2 drafter on (%s, graphs captured in %.0f ms), the MTP layer stays as the fallback\n", o.draft_model.c_str(), df2.ms_capture);