Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

DeepSeek engine: the prompt path is chosen from the prompt's length alone (96 tokens and more through the chunked pass, windows of 16 below), one number for every box - the same prompt always takes the same path and gives the same bits (the old per-request timing prediction flipped the logits between runs and took 7 s on a 28-token prompt); the chunked pass is warmed up after the model is ready, on a thread, so the BLAS library's ~3.6 s start is not paid by the first prompt a person types; startup line; DS4_PICK=adaptive, DS4_PICK_MIN=N, DS4_WARM=0 as switches; rule test

Niko1221dfd1b0f2914b
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 37f146a8a..a04ab750a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -891,6 +891,9 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
add_executable(ds4_experts_rule_test tests/dsv41/experts_rule_test.cpp)
target_include_directories(ds4_experts_rule_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/dsv41)
target_compile_features(ds4_experts_rule_test PRIVATE cxx_std_17)
+ add_executable(ds4_pick_rule_test tests/dsv41/pick_rule_test.cpp)
+ target_include_directories(ds4_pick_rule_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/dsv41)
+ target_compile_features(ds4_pick_rule_test PRIVATE cxx_std_17)
endif()
if(STRATA_ENABLE_HIP)
foreach(_t strata_dsv41_kernels strata-dsv41)
diff --git a/src/dsv41/ds4_main.cpp b/src/dsv41/ds4_main.cpp
index 857df4128..c01358278 100644
--- a/src/dsv41/ds4_main.cpp
+++ b/src/dsv41/ds4_main.cpp
@@ -1297,6 +1297,7 @@ int main(int argc, char** argv) {
return 1;
}
if (!o.vision_only) std::fprintf(stderr, "%s", m.describe().c_str());
+ if (const char* idle = std::getenv("DS4_TEST_IDLE_MS")) std::this_thread::sleep_for(std::chrono::milliseconds(std::atoi(idle))); // (tests: a person typing)
if (serve_mode) return serve(m, eos, spec, o.dspark_conf, o.pack);
const int window_given = window;
if (window == 0) window = 16; // (the scoring / profiling runs; the plain run below takes the model's feed unit)
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index b4671266c..ca7b3e011 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -13,6 +13,7 @@
// h = post * moe + comb^T h; pre = f_pre
// and after the last layer logits = head(rmsnorm(sum_s pre[s] h[s])).
#include "ds4_model.hpp"
+#include "ds4_pick_rule.hpp"
#include "ds4_experts_rule.hpp"
#include "ds4_engram_io.hpp"
@@ -1240,6 +1241,8 @@ struct Model::Impl {
}
~Impl() {
+ quit.store(true);
+ if (warm_bg.joinable()) warm_bg.join(); // (the warm-up uses the BLAS handles destroyed below)
eg_io.stop(); // (its threads write into h_eg)
for (auto& w : workers) w->stop();
for (size_t i = 0; i < chunks.size(); ++i)
@@ -2813,7 +2816,9 @@ struct Model::Impl {
init_store(); // (last: the store's arena is registered with the driver - page-locked memory the other pinned allocations must come before)
preload_store();
if (o.verbose) std::fprintf(stderr, "ds4: loaded in %.1f s\n%s", (now_ms() - t0) / 1000.0, describe().c_str());
+ if (chunk > 0) (void) pick_flow_min(); // (the startup line)
reset();
+ start_warm();
return true;
}
@@ -2824,6 +2829,47 @@ struct Model::Impl {
size_t ckv_rb() const { return kv_row_bytes(kKvFp4, g.hd); }
size_t ik_rb() const { return kv_row_bytes(kKvFp4p, g.ix_hd); }
+ // The chunked prompt's first GEMM of each kind pays the BLAS library's start (its handle's heuristics, the code objects of its
+ // kernels loaded on first use): ~1 s of hipModuleLoad plus ~2.5 s of library time on the R9700 (rocprofv3), 1-2 s on CUDA, once a
+ // process - 3.6 s on the first prompt that went through the flow, measured as 4.9 s for a 25-token pass against 1.0 s for the
+ // next one. Paid here instead, at load, on a thread beside the store's read: the chunk GEMMs on tiny inputs into the chunk's own
+ // scratch buffers (the mHC mix GEMM in F32, the FP16 GEMM the projections and the indexer weights use) - no engine state is
+ // touched (heat, profile, stats, positions, KV), so the answers are what they were. DS4_WARM=0 turns it off.
+ void warm_flow() {
+ const int T = 16, D = g.dim, hcd = g.hc * g.dim, n_mix = (2 + g.hc) * g.hc;
+ const float one = 1.0f, zero = 0.0f;
+ for (size_t di = 0; di < chunks.size(); ++di) {
+ if (!chunks[di] || chunks[di]->blas == nullptr) continue;
+ Chunk& c = *chunks[di];
+ if (c.C < T) continue;
+ cudaSetDevice(devs[di].id);
+ (void) cublasSgemm(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, n_mix, T, hcd, &one, c.h, hcd, c.h, hcd, &zero, c.mix, 32);
+ const int ns[2] = {256, 32};
+ for (int n : ns)
+ (void) cublasGemmEx(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, n, T, D, &one, c.w16, CUDA_R_16F, D, c.x16, CUDA_R_16F, D, &zero,
+ c.y, CUDA_R_32F, n, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP);
+ (void) cudaStreamSynchronize(devs[di].st);
+ (void) cudaGetLastError();
+ }
+ }
+ // The warm-up runs once the model is loaded, on a thread of its own: HIP serialises the BLAS start with the store's page-locked
+ // arena (measured: beside the store's read it finished only when the read did, 21 s, and load still grew by its 3.5 s), so
+ // it cannot hide inside the load; after it, a person typing gives it the seconds it needs. A chunked pass that starts first
+ // waits for it (warm_mu) - never later than that pass would have been without it - and a warm-up that finds the flow already
+ // used does nothing. DS4_WARM=0 turns it off.
+ std::mutex warm_mu;
+ std::atomic<bool> flow_used{false};
+ std::thread warm_bg;
+ void start_warm() {
+ if (chunk <= 0 || (std::getenv("DS4_WARM") && std::atoi(std::getenv("DS4_WARM")) == 0)) return;
+ warm_bg = std::thread([this] {
+ std::lock_guard<std::mutex> lk(warm_mu);
+ if (flow_used.load() || quit.load()) return;
+ const double tw = now_ms();
+ warm_flow();
+ if (o.verbose) std::fprintf(stderr, "ds4: prompt path warmed up in %.0f ms\n", now_ms() - tw);
+ });
+ }
// ---- the chunked prompt path
size_t x_cap() const { return (size_t) std::max<int>(kWin, chunk + 2); } // a window's / chunk's new source rows
size_t chunk_hand_bytes() const {
@@ -7873,8 +7919,28 @@ struct Model::Impl {
// this PC's speeds - the flow's times what it really took against its predictions so far for prompts of that size
// (pm_k), windows' once some ran what a window took times the windows; the flow unless windows are predicted
// faster (by half, with less than kPickRes in RAM); said with --verbose
+ // The choice is made from the prompt's length alone (pick_flow_min): the same prompt takes the same path - and so
+ // gives the same bits, the two paths round differently - in every session and on every restart. The timing prediction below
+ // (DS4_PICK=adaptive) used measured speeds and sampled page residency, so it flipped between runs: r9700, one 16-token feed
+ // through the flow in one run, another feed in the next, three different logits checksums in six runs. Measured on 4 and 1 R9700
+ // (first prompt of a run, windows against the flow, ms): 28 tokens 594 / 4966 (4 cards), 1094 / 5588 (1 card); 100: 1164 / 5375,
+ // 3279 / 7299; 300: 2831 / 5874, 9018 / 9803; 600: 5304 / 6808, 16969 / 11327; 1500: 13325 / 8012, 41547 / 18187 - the flow pays a
+ // fixed ~5 s the first time (its buffers and GEMM libraries), then wins from about 400 tokens up.
+ // The length from which the flow is taken: one number for every box (ds4_pick_rule.hpp says why 96), DS4_PICK_MIN=N overrides.
+ int pick_flow_min() {
+ if (pick_min_ > 0) return pick_min_;
+ pick_min_ = prompt_flow_min(std::getenv("DS4_PICK_MIN"));
+ std::fprintf(stderr, "ds4: prompt path: the chunked pass from %d tokens, windows of 16 below\n", pick_min_);
+ return pick_min_;
+ }
+ int pick_min_ = 0;
bool pick_path(int n) {
+ static const bool adaptive = [] {
+ const char* e = std::getenv("DS4_PICK");
+ return e != nullptr && std::strcmp(e, "adaptive") == 0;
+ }();
pm.last = -1;
+ if (!adaptive) return n >= pick_flow_min();
if (!pm.on || n >= kPickMax) return true;
double f_res = 0, f_cold = 0, w = 0;
const int nw = n / kWin, rem = n % kWin;
@@ -8069,6 +8135,8 @@ struct Model::Impl {
// the last min(logits_rows, 16) rows (logits_rows > 16: a run of one chunk, every row's into chunk_logits)
bool forward_chunks(const int* tk, int n, int logits_rows, std::string& err, int snap_slot = -1) {
if (n < 1) return true;
+ flow_used.store(true);
+ std::lock_guard<std::mutex> warm_lk(warm_mu); // (the warm-up, if it is running, finishes first)
if (pos + n > o.max_context) {
err = "the context is full";
return false;
diff --git a/src/dsv41/ds4_pick_rule.hpp b/src/dsv41/ds4_pick_rule.hpp
new file mode 100644
index 000000000..5534c7021
--- /dev/null
+++ b/src/dsv41/ds4_pick_rule.hpp
@@ -0,0 +1,29 @@
+// Strata. Which way the DeepSeek V4.1 engine reads a prompt: the chunked pass (the "flow": each chunk goes through every layer at
+// once, the experts the GPUs do not hold streamed to the GPU once a chunk) or windows of 16 tokens (the CPU lane computes the experts
+// it holds, for every window again). The choice is the prompt's length against one number, the same on every box and in every session,
+// so a prompt always takes the same path and gives the same bits (the two paths round differently).
+//
+// Why 96. First prompt of a fresh process, the chunked pass warmed up (the BLAS library's start paid while the user types), r9700
+// 2.00 pack, milliseconds windows / flow: 1 card 28: 1144 / 2252, 100: 3384 / 3922, 300: 9310 / 7281, 600: 17349 / 8165, 1500: 42469 / 10920;
+// 4 cards 28: 585 / 922, 100: 1161 / 1264, 300: 2837 / 1923, 600: 5300 / 2675, 1500: 13353 / 4181 - the crossover is ~120-140 tokens
+// there. Boxes whose experts come mostly from an SSD or a slower CPU lane cross earlier (P2003 RTX 3060, no warm-up: 28: 3322 / 4063,
+// 100: 8893 / 6817, 300: 26137 / 11417; the owner's RTX 5070 ~100; p620 100: 11839 / 7580), so a box-independent 96 costs the
+// fast-CPU boxes at most 0.5 s on a 100-token prompt and saves the slow ones 2-4 s; 600 tokens and up take the flow everywhere.
+#pragma once
+
+#include <cstdlib>
+
+namespace strata::ds4 {
+
+constexpr int kPromptFlowMin = 96; // tokens from which a prompt goes through the chunked pass
+
+// env: DS4_PICK_MIN (null: the default)
+inline int prompt_flow_min(const char* env) {
+ if (env == nullptr || *env == 0) return kPromptFlowMin;
+ const int v = std::atoi(env);
+ return v < 1 ? 1 : v;
+}
+
+inline bool prompt_uses_flow(int tokens, int flow_min) { return tokens >= flow_min; }
+
+} // namespace strata::ds4
diff --git a/tests/dsv41/pick_rule_test.cpp b/tests/dsv41/pick_rule_test.cpp
new file mode 100644
index 000000000..2b8deba62
--- /dev/null
+++ b/tests/dsv41/pick_rule_test.cpp
@@ -0,0 +1,38 @@
+// Strata. The DeepSeek V4.1 engine's prompt-path rule (src/dsv41/ds4_pick_rule.hpp): by length only, one number, DS4_PICK_MIN moves it.
+#include "ds4_pick_rule.hpp"
+
+#include <cstdio>
+
+using strata::ds4::prompt_flow_min;
+using strata::ds4::prompt_uses_flow;
+
+static int fails = 0;
+#define CHECK(c) \
+ do { \
+ if (!(c)) { \
+ std::printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #c); \
+ ++fails; \
+ } \
+ } while (0)
+
+int main() {
+ const int m = prompt_flow_min(nullptr);
+ CHECK(m == 96);
+ CHECK(prompt_flow_min("") == 96);
+ CHECK(prompt_flow_min("400") == 400);
+ CHECK(prompt_flow_min("2") == 2);
+ CHECK(prompt_flow_min("0") == 1); // never below one token
+ CHECK(prompt_flow_min("-5") == 1);
+ // the same prompt always takes the same path
+ for (int rep = 0; rep < 3; ++rep) {
+ CHECK(!prompt_uses_flow(1, m)); // a decode token
+ CHECK(!prompt_uses_flow(28, m)); // a short chat turn
+ CHECK(!prompt_uses_flow(95, m));
+ CHECK(prompt_uses_flow(96, m));
+ CHECK(prompt_uses_flow(300, m));
+ CHECK(prompt_uses_flow(8192, m));
+ }
+ CHECK(prompt_uses_flow(100, 2) && !prompt_uses_flow(100, 100000)); // DS4_PICK_MIN=2 / 100000: always flow / always windows
+ if (fails == 0) std::printf("prompt path rule: ok\n");
+ return fails == 0 ? 0 : 1;
+}