Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41 win: --dspark-block N (at most N drafts a round)

Niko1221ec5b2bdc4e9e
diff --git a/src/dsv41/ds4_main.cpp b/src/dsv41/ds4_main.cpp
index 73389d71f..e614e6cb1 100644
--- a/src/dsv41/ds4_main.cpp
+++ b/src/dsv41/ds4_main.cpp
@@ -74,7 +74,7 @@ void usage() {
std::fprintf(stderr,
"maya-ds4 --pack <model>/maya-ds4 [--gpus 0,1,...] [--max-context N] [--reserve-mib N]\n"
" [--vram-experts F] [--cpu-threads N] [--expert-profile FILE|none] [--ssd-cache GB] [--ssd-policy lru|lfu|lfulru] [--no-expert-cache] [--verbose]\n"
- " [--tp N] [--threads on|off] [--prefill-chunk N] [--dspark on|off|none] [--dspark-conf F]\n"
+ " [--tp N] [--threads on|off] [--prefill-chunk N] [--dspark on|off|none|auto] [--dspark-block N] [--dspark-conf F]\n"
" [--vision on|off] [--mmap-experts on|off|auto]\n"
" (--ids I,J,... [--images FILE] [--n N] [--dump-logits FILE] | --serve\n"
" | --encode-image FILE.rgb --size WxH --dump-image OUT\n"
@@ -781,6 +781,7 @@ int main(int argc, char** argv) {
o.dspark = v != "none"; // none: not even loaded
o.dspark_auto = v == "auto";
} else if (a == "--dspark-conf") o.dspark_conf = (float) std::atof(val("--dspark-conf").c_str());
+ else if (a == "--dspark-block") o.dspark_block = std::max(0, std::atoi(val("--dspark-block").c_str()));
else if (a == "--tp") o.tp = std::max(1, std::atoi(val("--tp").c_str()));
else if (a == "--prefill-chunk") o.prefill_chunk = std::max(0, std::atoi(val("--prefill-chunk").c_str()));
else if (a == "--threads") o.threads = val("--threads") != "off";
@@ -930,6 +931,8 @@ int main(int argc, char** argv) {
(long long) s0.eg_rows, 100.0 * (double) s0.eg_hits / (double) s0.eg_rows, (long long) s0.eg_reads);
const long long io0 = io_read_bytes();
std::vector<int> out;
+ std::vector<double> tok_ms; // (DS4_TOKLOG: each token's time since the one before - the latency ledger)
+ double t_prev = t1;
unsigned long long counter = 0;
long long d_off = 0, d_acc = 0, rounds = 0;
bool cool = false;
@@ -938,6 +941,11 @@ int main(int argc, char** argv) {
std::vector<int> got;
for (;;) {
out.push_back(tok);
+ {
+ const double tn = now_ms();
+ tok_ms.push_back(tn - t_prev);
+ t_prev = tn;
+ }
std::printf("%d,", tok);
std::fflush(stdout);
if ((tok == 1 && std::getenv("DS4_NO_EOS") == nullptr) || (int) out.size() >= n) break; // (DS4_NO_EOS: decode n tokens whatever)
@@ -970,6 +978,13 @@ int main(int argc, char** argv) {
s1.verify_ms - s0.verify_ms, (long long) (s1.drafts_offered > 0 ? 0 : 0));
std::fprintf(stderr, "dspark %s\n", per_position(s0, s1).c_str());
}
+ if (std::getenv("DS4_TOKLOG") != nullptr && tok_ms.size() > 8) { // the decode's tokens after the first (which carries the prompt's tail)
+ std::vector<double> v(tok_ms.begin() + 1, tok_ms.end());
+ std::sort(v.begin(), v.end());
+ std::fprintf(stderr, "token latency ms: p50 %.0f p95 %.0f p99 %.0f max %.0f mean %.0f (%zu tokens)\n", v[v.size() / 2],
+ v[(size_t) ((double) v.size() * 0.95)], v[(size_t) ((double) v.size() * 0.99)], v.back(),
+ std::accumulate(v.begin(), v.end(), 0.0) / (double) v.size(), v.size());
+ }
const double t2 = now_ms();
const auto& s = m.stats();
const double nt = std::max<double>(1.0, (double) (s.tokens - s0.tokens));
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index 716802b22..a9e370441 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -1942,6 +1942,7 @@ struct Model::Impl {
}
}
if (!o.dspark) g.n_mtp = 0;
+ if (o.dspark_block > 0 && std::getenv("DS4_DSPARK_MAX") == nullptr) max_drafts = o.dspark_block; // (--dspark-block N: at most N drafts a round)
L.resize((size_t) (g.nl + g.n_mtp));
int last_src = -1;
for (int il = 0; il < g.nl + g.n_mtp; ++il) {
diff --git a/src/dsv41/ds4_model.hpp b/src/dsv41/ds4_model.hpp
index dc5925cf4..929636d53 100644
--- a/src/dsv41/ds4_model.hpp
+++ b/src/dsv41/ds4_model.hpp
@@ -42,6 +42,7 @@ struct Options {
// trunk's hottest would have had); --dspark on / off / none says it outright
bool dspark_auto = true;
bool dspark = true; // load the DSpark drafter (the checkpoint's mtp.* blocks) for speculative decoding
+ int dspark_block = 0; // at most this many drafts a round (--dspark-block N; 0: the pack's block size)
float dspark_conf = 0.5f; // a draft is verified while its confidence (sigmoid) is at least this
// tensor parallelism: runs of tp GPUs of one kind (the same memory) take the same layers, each a share of the
// heads, of the output projection and of every expert's FFN, summed through host-mapped memory (a GPU unlike the