Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41: --pack-info (the pack as a description: type map, bytes an expert/a token, tiers, tensors with no GPU kernel)

Niko12219ee8b0cf795f
diff --git a/CMakeLists.txt b/CMakeLists.txt
index ff5409244..e07c2462f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -777,7 +777,7 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
target_link_libraries(strata_dsv41_kernels PUBLIC ${_strata_gpu_runtime_target})
add_executable(strata-dsv41 src/dsv41/ds4_main.cpp src/dsv41/ds4_model.cu src/dsv41/ds4_kernels.cu src/dsv41/ds4_vision.cu
src/dsv41/ds4_gguf.cu src/dsv41/ds4_gguf_cpu.cpp src/dsv41/ds4_iq.cu src/dsv41/ds4_iq_cpu.cpp
- src/dsv41/ds4_tq.cu src/dsv41/ds4_tq_cpu.cpp src/dsv41/ds4_gg_cpu.cpp)
+ src/dsv41/ds4_tq.cu src/dsv41/ds4_tq_cpu.cpp src/dsv41/ds4_gg_cpu.cpp src/dsv41/ds4_packinfo.cpp)
target_include_directories(strata-dsv41 PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories}
${CMAKE_CURRENT_SOURCE_DIR}/third_party/ggml)
target_link_libraries(strata-dsv41 PRIVATE strata_dsv41_kernels ${_strata_gpu_runtime_target} ${_strata_gpu_blas_target})
diff --git a/src/dsv41/ds4_main.cpp b/src/dsv41/ds4_main.cpp
index 9c8956316..889f30405 100644
--- a/src/dsv41/ds4_main.cpp
+++ b/src/dsv41/ds4_main.cpp
@@ -13,6 +13,7 @@
// that prompt's end; anything else starts over. `GENI <max_new> [keys] <file> <ids>` carries the prompt's pictures
// (the file: see read_pics); the state is shared only where the same picture sits at the same place.
#include "ds4_model.hpp"
+#include "ds4_packinfo.hpp"
#include <algorithm>
#include <cmath>
@@ -74,7 +75,8 @@ void usage() {
" [--tp N] [--threads on|off] [--prefill-chunk N] [--dspark on|off|none] [--dspark-conf F]\n"
" [--vision on|off] [--mmap-experts on|off|auto]\n"
" (--ids I,J,... [--images FILE] [--n N] [--dump-logits FILE] | --serve\n"
- " | --encode-image FILE.rgb --size WxH --dump-image OUT)\n");
+ " | --encode-image FILE.rgb --size WxH --dump-image OUT\n"
+ " | --pack-info)\n");
}
// stdin lines on a thread, so a request can see STOP while it decodes
@@ -643,6 +645,7 @@ int main(int argc, char** argv) {
float temp = 0.0f; // the CLI's sampling (0: greedy)
unsigned long long cli_seed = 1;
bool serve_mode = false;
+ bool pack_info_mode = false;
std::string images, enc_rgb, enc_out;
std::string score_ids, score_out;
int enc_w = 0, enc_h = 0;
@@ -695,6 +698,7 @@ int main(int argc, char** argv) {
else if (a == "--score") score_ids = val("--score");
else if (a == "--score-out") score_out = val("--score-out");
else if (a == "--serve") serve_mode = true;
+ else if (a == "--pack-info") pack_info_mode = true;
else if (a == "--layer-split") (void) val("--layer-split"); // the server's (this engine balances its own split)
else if (a == "--help" || a == "-h") {
usage();
@@ -705,6 +709,13 @@ int main(int argc, char** argv) {
return 2;
}
}
+ if (pack_info_mode) { // the pack's description only: no device, no weights, no lock
+ if (o.pack.empty()) {
+ usage();
+ return 2;
+ }
+ return strata::ds4::pack_info(o.pack, stdout);
+ }
if (o.pack.empty() || (!serve_mode && ids_s.empty() && enc_rgb.empty() && score_ids.empty())) {
usage();
return 2;
diff --git a/src/dsv41/ds4_packinfo.cpp b/src/dsv41/ds4_packinfo.cpp
new file mode 100644
index 000000000..61e1c8bd1
--- /dev/null
+++ b/src/dsv41/ds4_packinfo.cpp
@@ -0,0 +1,191 @@
+// Strata. `strata-dsv41 --pack-info`: the pack as a description, before anything is loaded onto a GPU - the type map
+// (layer x role), the bytes an expert and a token, the tier sizes, and the tensors whose type has no device kernel (they
+// would go to the CPU lane, or stop the load). Host only, no device, no weights read: tensors.txt and ds4.txt are the
+// pack's whole description (tools/dsv41/pack_gguf.py). The same report is the first line of every benchmark log.
+#include "ds4_packinfo.hpp"
+
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <fstream>
+#include <map>
+#include <set>
+#include <string>
+#include <vector>
+
+namespace strata::ds4 {
+namespace {
+
+// The ggml block types the device kernels take (ds4_iq.cu's Ty<> / DS4_IQ_SWITCH + ds4_gguf.cu's MXFP4). A name here
+// and not in the pack's tensor list is fine; a tensor whose type is not here is the CPU lane's (ggml-cpu's row dots).
+bool device_type(const std::string& t) {
+ static const std::set<std::string> ok = {"MXFP4", "Q2_K", "Q3_K", "Q4_K", "Q5_K", "Q6_K", "IQ2_XXS", "IQ2_XS",
+ "IQ2_S", "IQ3_XXS", "IQ3_S", "IQ4_XS", "TQK6", "TQK7", "TQK5X", "TQK4X",
+ "TQK5T", "TQK4T"};
+ return ok.count(t) != 0;
+}
+// The types the dense path reads as they are (ds4_gguf.cu: BF16 / F32 / MXFP8 rows, Q8_0 rows; ds4_iq's GEMV for the
+// K-quants is the same device support as the experts').
+bool dense_type(const std::string& t) {
+ return t == "BF16" || t == "F32" || t == "F16" || t == "MXFP8" || t == "Q8_0" || device_type(t);
+}
+
+struct Tensor {
+ std::string dtype;
+ double bytes = 0;
+ std::vector<long long> shape;
+};
+
+double gb(double b) { return b / 1e9; }
+
+} // namespace
+
+int pack_info(const std::string& dir, std::FILE* out) {
+ std::map<std::string, std::string> kv;
+ {
+ std::ifstream f(dir + "/ds4.txt");
+ if (!f) {
+ std::fprintf(out, "pack-info: %s/ds4.txt: not found\n", dir.c_str());
+ return 1;
+ }
+ std::string line;
+ while (std::getline(f, line)) {
+ const size_t sp = line.find(' ');
+ if (sp != std::string::npos) kv[line.substr(0, sp)] = line.substr(sp + 1);
+ }
+ }
+ auto num = [&](const char* k, long long d) {
+ auto it = kv.find(k);
+ return it == kv.end() ? d : std::atoll(it->second.c_str());
+ };
+ const long long nl = num("num_hidden_layers", 0), ne = num("n_routed_experts", 0), topk = num("num_experts_per_tok", 0);
+ const long long dim = num("hidden_size", 0), ff = num("moe_intermediate_size", 0), vocab = num("vocab_size", 0);
+ std::map<std::string, Tensor> t;
+ {
+ std::FILE* f = std::fopen((dir + "/tensors.txt").c_str(), "r");
+ if (f == nullptr) {
+ std::fprintf(out, "pack-info: %s/tensors.txt: not found\n", dir.c_str());
+ return 1;
+ }
+ std::vector<char> buf(1 << 16);
+ while (std::fgets(buf.data(), (int) buf.size(), f) != nullptr) {
+ char* s = buf.data();
+ if (s[0] != 't') continue;
+ char* save = nullptr;
+ ::strtok_r(s, " \n", &save);
+ const char* name = ::strtok_r(nullptr, " \n", &save);
+ const char* dt = ::strtok_r(nullptr, " \n", &save);
+ (void) ::strtok_r(nullptr, " \n", &save); // file
+ (void) ::strtok_r(nullptr, " \n", &save); // offset
+ const char* nb = ::strtok_r(nullptr, " \n", &save);
+ if (name == nullptr || dt == nullptr || nb == nullptr) continue;
+ Tensor x;
+ x.dtype = dt;
+ x.bytes = (double) std::strtoull(nb, nullptr, 10);
+ while (const char* d = ::strtok_r(nullptr, " \n", &save)) x.shape.push_back(std::atoll(d));
+ t[name] = std::move(x);
+ }
+ std::fclose(f);
+ }
+ if (t.empty()) {
+ std::fprintf(out, "pack-info: %s/tensors.txt: no tensors\n", dir.c_str());
+ return 1;
+ }
+
+ // ---- the expert map: layer -> role types and bytes an expert
+ struct Role {
+ std::string type[3];
+ double bytes[3] = {0, 0, 0};
+ bool mixed = false;
+ int n = 0;
+ };
+ std::vector<Role> lay((size_t) nl);
+ std::map<std::string, long long> type_count; // tensor count a type (the experts' 3 roles)
+ double experts_bytes = 0;
+ std::set<std::string> no_kernel;
+ for (long long il = 0; il < nl; ++il) {
+ Role& r = lay[(size_t) il];
+ const std::string pre = "layers." + std::to_string(il) + ".ffn.experts.";
+ for (long long e = 0; e < ne; ++e) {
+ const char* role[3] = {"w1", "w3", "w2"};
+ for (int k = 0; k < 3; ++k) {
+ auto it = t.find(pre + std::to_string(e) + "." + role[k] + ".weight");
+ if (it == t.end()) {
+ std::fprintf(out, "pack-info: %s%s%lld.%s.weight: missing\n", pre.c_str(), "", e, role[k]);
+ return 1;
+ }
+ if (e == 0) {
+ r.type[k] = it->second.dtype;
+ r.bytes[k] = it->second.bytes;
+ } else if (it->second.dtype != r.type[k] || it->second.bytes != r.bytes[k]) {
+ r.mixed = true;
+ }
+ if (e == 0) {
+ ++type_count[it->second.dtype];
+ if (!device_type(it->second.dtype)) no_kernel.insert("layer " + std::to_string(il) + " " + role[k] + " " + it->second.dtype);
+ }
+ }
+ }
+ r.n = (int) ne;
+ experts_bytes += (double) ne * (r.bytes[0] + r.bytes[1] + r.bytes[2]);
+ }
+ const double per_expert = nl > 0 ? experts_bytes / ((double) nl * (double) ne) : 0;
+ // (a token picks `topk` experts a layer: sum over the layers of topk x one expert of that layer)
+ double token_bytes = 0;
+ for (long long il = 0; il < nl; ++il) token_bytes += (double) topk * (lay[(size_t) il].bytes[0] + lay[(size_t) il].bytes[1] + lay[(size_t) il].bytes[2]);
+ const double expert_params = (double) nl * (double) ne * 3.0 * (double) ff * (double) dim;
+
+ // ---- the dense side: everything that is not an expert tensor
+ double dense = 0, dense_embed = 0, dense_head = 0, dense_mtp = 0, dense_vision = 0, dense_engram = 0;
+ std::set<std::string> dense_types;
+ for (const auto& [name, x] : t) {
+ if (name.find(".ffn.experts.") != std::string::npos && name.rfind("layers.", 0) == 0) continue;
+ dense += x.bytes;
+ dense_types.insert(x.dtype);
+ if (name == "embed.weight") dense_embed += x.bytes;
+ else if (name == "head.weight") dense_head += x.bytes;
+ else if (name.rfind("mtp.", 0) == 0) dense_mtp += x.bytes;
+ else if (name.find(".engram.") != std::string::npos) dense_engram += x.bytes;
+ else if (name.rfind("vision.", 0) == 0 || name.rfind("aligner.", 0) == 0) dense_vision += x.bytes;
+ if (!dense_type(x.dtype)) no_kernel.insert("dense " + name + " " + x.dtype);
+ }
+
+ // ---- report
+ std::fprintf(out, "pack-info %s\n", dir.c_str());
+ std::fprintf(out, " model_dir %s format %s %lld layers x %lld experts, %lld used; dim %lld, ff %lld, vocab %lld\n",
+ kv.count("model_dir") ? kv["model_dir"].c_str() : "?", kv.count("format") ? kv["format"].c_str() : "?", nl, ne, topk,
+ dim, ff, vocab);
+ std::fprintf(out, " layer gate+up down bytes/expert MB/token\n");
+ for (long long il = 0; il < nl; ++il) {
+ const Role& r = lay[(size_t) il];
+ const double eb = r.bytes[0] + r.bytes[1] + r.bytes[2];
+ std::fprintf(out, " %5lld %-9s %-9s %13.0f %10.2f%s\n", il, r.type[0].c_str(), r.type[2].c_str(), eb,
+ (double) topk * eb / 1e6, r.mixed ? " (mixed)" : "");
+ }
+ std::fprintf(out, " expert tensors by type:");
+ for (const auto& [ty, n] : type_count) std::fprintf(out, " %s %lld", ty.c_str(), n);
+ std::fprintf(out, "\n");
+ std::fprintf(out, " an expert %.2f MB, %lld picks a token: %.3f GB a token\n", per_expert / 1e6, topk, token_bytes / 1e9);
+ std::fprintf(out, " all experts %.1f GB at %.2f bpw (%.0f G parameters), dense %.2f GB (embed %.2f, head %.2f, mtp %.2f, vision %.2f, engram %.2f)\n",
+ gb(experts_bytes), expert_params > 0 ? experts_bytes * 8.0 / expert_params : 0.0, expert_params / 1e9, gb(dense),
+ gb(dense_embed), gb(dense_head), gb(dense_mtp), gb(dense_vision), gb(dense_engram));
+ std::fprintf(out, " dense types:");
+ for (const auto& d : dense_types) std::fprintf(out, " %s", d.c_str());
+ std::fprintf(out, "\n");
+ if (no_kernel.empty()) {
+ std::fprintf(out, " no GPU kernel: none (every tensor is a device type)\n");
+ } else {
+ std::fprintf(out, " no GPU kernel (%zu): the CPU lane takes them, they cannot go in the VRAM tier\n", no_kernel.size());
+ int shown = 0;
+ for (const std::string& s : no_kernel) {
+ std::fprintf(out, " %s\n", s.c_str());
+ if (++shown >= 40) {
+ std::fprintf(out, " ... %zu more\n", no_kernel.size() - (size_t) shown);
+ break;
+ }
+ }
+ }
+ return 0;
+}
+
+} // namespace strata::ds4
diff --git a/src/dsv41/ds4_packinfo.hpp b/src/dsv41/ds4_packinfo.hpp
new file mode 100644
index 000000000..c7597eebf
--- /dev/null
+++ b/src/dsv41/ds4_packinfo.hpp
@@ -0,0 +1,13 @@
+// Strata. The pack report (ds4_packinfo.cpp): `strata-dsv41 --pack-info --pack <dir>`. Host only: no device, no weights.
+#pragma once
+
+#include <cstdio>
+#include <string>
+
+namespace strata::ds4 {
+
+/// Prints the pack's type map, its bytes an expert / a token, its tier sizes and the tensors whose type has no device
+/// kernel. 0 when the pack's description was read, non-zero (and a line saying what is missing) otherwise.
+int pack_info(const std::string& dir, std::FILE* out);
+
+} // namespace strata::ds4