Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41: --pack-info (the pack as a description: type map, bytes an expert/a token, tiers, tensors with no GPU kernel)
Niko12219ee8b0cf795f
diff --git a/CMakeLists.txt b/CMakeLists.txtindex ff5409244..e07c2462f 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -777,7 +777,7 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)target_link_libraries(strata_dsv41_kernels PUBLIC ${_strata_gpu_runtime_target})add_executable(strata-dsv41 src/dsv41/ds4_main.cpp src/dsv41/ds4_model.cu src/dsv41/ds4_kernels.cu src/dsv41/ds4_vision.cusrc/dsv41/ds4_gguf.cu src/dsv41/ds4_gguf_cpu.cpp src/dsv41/ds4_iq.cu src/dsv41/ds4_iq_cpu.cpp- src/dsv41/ds4_tq.cu src/dsv41/ds4_tq_cpu.cpp src/dsv41/ds4_gg_cpu.cpp)+ src/dsv41/ds4_tq.cu src/dsv41/ds4_tq_cpu.cpp src/dsv41/ds4_gg_cpu.cpp src/dsv41/ds4_packinfo.cpp)target_include_directories(strata-dsv41 PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories}${CMAKE_CURRENT_SOURCE_DIR}/third_party/ggml)target_link_libraries(strata-dsv41 PRIVATE strata_dsv41_kernels ${_strata_gpu_runtime_target} ${_strata_gpu_blas_target})diff --git a/src/dsv41/ds4_main.cpp b/src/dsv41/ds4_main.cppindex 9c8956316..889f30405 100644--- a/src/dsv41/ds4_main.cpp+++ b/src/dsv41/ds4_main.cpp@@ -13,6 +13,7 @@// that prompt's end; anything else starts over. `GENI <max_new> [keys] <file> <ids>` carries the prompt's pictures// (the file: see read_pics); the state is shared only where the same picture sits at the same place.#include "ds4_model.hpp"+#include "ds4_packinfo.hpp"#include <algorithm>#include <cmath>@@ -74,7 +75,8 @@ void usage() {" [--tp N] [--threads on|off] [--prefill-chunk N] [--dspark on|off|none] [--dspark-conf F]\n"" [--vision on|off] [--mmap-experts on|off|auto]\n"" (--ids I,J,... [--images FILE] [--n N] [--dump-logits FILE] | --serve\n"- " | --encode-image FILE.rgb --size WxH --dump-image OUT)\n");+ " | --encode-image FILE.rgb --size WxH --dump-image OUT\n"+ " | --pack-info)\n");}// stdin lines on a thread, so a request can see STOP while it decodes@@ -643,6 +645,7 @@ int main(int argc, char** argv) {float temp = 0.0f; // the CLI's sampling (0: greedy)unsigned long long cli_seed = 1;bool serve_mode = false;+ bool pack_info_mode = false;std::string images, enc_rgb, enc_out;std::string score_ids, score_out;int enc_w = 0, enc_h = 0;@@ -695,6 +698,7 @@ int main(int argc, char** argv) {else if (a == "--score") score_ids = val("--score");else if (a == "--score-out") score_out = val("--score-out");else if (a == "--serve") serve_mode = true;+ else if (a == "--pack-info") pack_info_mode = true;else if (a == "--layer-split") (void) val("--layer-split"); // the server's (this engine balances its own split)else if (a == "--help" || a == "-h") {usage();@@ -705,6 +709,13 @@ int main(int argc, char** argv) {return 2;}}+ if (pack_info_mode) { // the pack's description only: no device, no weights, no lock+ if (o.pack.empty()) {+ usage();+ return 2;+ }+ return strata::ds4::pack_info(o.pack, stdout);+ }if (o.pack.empty() || (!serve_mode && ids_s.empty() && enc_rgb.empty() && score_ids.empty())) {usage();return 2;diff --git a/src/dsv41/ds4_packinfo.cpp b/src/dsv41/ds4_packinfo.cppnew file mode 100644index 000000000..61e1c8bd1--- /dev/null+++ b/src/dsv41/ds4_packinfo.cpp@@ -0,0 +1,191 @@+// Strata. `strata-dsv41 --pack-info`: the pack as a description, before anything is loaded onto a GPU - the type map+// (layer x role), the bytes an expert and a token, the tier sizes, and the tensors whose type has no device kernel (they+// would go to the CPU lane, or stop the load). Host only, no device, no weights read: tensors.txt and ds4.txt are the+// pack's whole description (tools/dsv41/pack_gguf.py). The same report is the first line of every benchmark log.+#include "ds4_packinfo.hpp"++#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <fstream>+#include <map>+#include <set>+#include <string>+#include <vector>++namespace strata::ds4 {+namespace {++// The ggml block types the device kernels take (ds4_iq.cu's Ty<> / DS4_IQ_SWITCH + ds4_gguf.cu's MXFP4). A name here+// and not in the pack's tensor list is fine; a tensor whose type is not here is the CPU lane's (ggml-cpu's row dots).+bool device_type(const std::string& t) {+ static const std::set<std::string> ok = {"MXFP4", "Q2_K", "Q3_K", "Q4_K", "Q5_K", "Q6_K", "IQ2_XXS", "IQ2_XS",+ "IQ2_S", "IQ3_XXS", "IQ3_S", "IQ4_XS", "TQK6", "TQK7", "TQK5X", "TQK4X",+ "TQK5T", "TQK4T"};+ return ok.count(t) != 0;+}+// The types the dense path reads as they are (ds4_gguf.cu: BF16 / F32 / MXFP8 rows, Q8_0 rows; ds4_iq's GEMV for the+// K-quants is the same device support as the experts').+bool dense_type(const std::string& t) {+ return t == "BF16" || t == "F32" || t == "F16" || t == "MXFP8" || t == "Q8_0" || device_type(t);+}++struct Tensor {+ std::string dtype;+ double bytes = 0;+ std::vector<long long> shape;+};++double gb(double b) { return b / 1e9; }++} // namespace++int pack_info(const std::string& dir, std::FILE* out) {+ std::map<std::string, std::string> kv;+ {+ std::ifstream f(dir + "/ds4.txt");+ if (!f) {+ std::fprintf(out, "pack-info: %s/ds4.txt: not found\n", dir.c_str());+ return 1;+ }+ std::string line;+ while (std::getline(f, line)) {+ const size_t sp = line.find(' ');+ if (sp != std::string::npos) kv[line.substr(0, sp)] = line.substr(sp + 1);+ }+ }+ auto num = [&](const char* k, long long d) {+ auto it = kv.find(k);+ return it == kv.end() ? d : std::atoll(it->second.c_str());+ };+ const long long nl = num("num_hidden_layers", 0), ne = num("n_routed_experts", 0), topk = num("num_experts_per_tok", 0);+ const long long dim = num("hidden_size", 0), ff = num("moe_intermediate_size", 0), vocab = num("vocab_size", 0);+ std::map<std::string, Tensor> t;+ {+ std::FILE* f = std::fopen((dir + "/tensors.txt").c_str(), "r");+ if (f == nullptr) {+ std::fprintf(out, "pack-info: %s/tensors.txt: not found\n", dir.c_str());+ return 1;+ }+ std::vector<char> buf(1 << 16);+ while (std::fgets(buf.data(), (int) buf.size(), f) != nullptr) {+ char* s = buf.data();+ if (s[0] != 't') continue;+ char* save = nullptr;+ ::strtok_r(s, " \n", &save);+ const char* name = ::strtok_r(nullptr, " \n", &save);+ const char* dt = ::strtok_r(nullptr, " \n", &save);+ (void) ::strtok_r(nullptr, " \n", &save); // file+ (void) ::strtok_r(nullptr, " \n", &save); // offset+ const char* nb = ::strtok_r(nullptr, " \n", &save);+ if (name == nullptr || dt == nullptr || nb == nullptr) continue;+ Tensor x;+ x.dtype = dt;+ x.bytes = (double) std::strtoull(nb, nullptr, 10);+ while (const char* d = ::strtok_r(nullptr, " \n", &save)) x.shape.push_back(std::atoll(d));+ t[name] = std::move(x);+ }+ std::fclose(f);+ }+ if (t.empty()) {+ std::fprintf(out, "pack-info: %s/tensors.txt: no tensors\n", dir.c_str());+ return 1;+ }++ // ---- the expert map: layer -> role types and bytes an expert+ struct Role {+ std::string type[3];+ double bytes[3] = {0, 0, 0};+ bool mixed = false;+ int n = 0;+ };+ std::vector<Role> lay((size_t) nl);+ std::map<std::string, long long> type_count; // tensor count a type (the experts' 3 roles)+ double experts_bytes = 0;+ std::set<std::string> no_kernel;+ for (long long il = 0; il < nl; ++il) {+ Role& r = lay[(size_t) il];+ const std::string pre = "layers." + std::to_string(il) + ".ffn.experts.";+ for (long long e = 0; e < ne; ++e) {+ const char* role[3] = {"w1", "w3", "w2"};+ for (int k = 0; k < 3; ++k) {+ auto it = t.find(pre + std::to_string(e) + "." + role[k] + ".weight");+ if (it == t.end()) {+ std::fprintf(out, "pack-info: %s%s%lld.%s.weight: missing\n", pre.c_str(), "", e, role[k]);+ return 1;+ }+ if (e == 0) {+ r.type[k] = it->second.dtype;+ r.bytes[k] = it->second.bytes;+ } else if (it->second.dtype != r.type[k] || it->second.bytes != r.bytes[k]) {+ r.mixed = true;+ }+ if (e == 0) {+ ++type_count[it->second.dtype];+ if (!device_type(it->second.dtype)) no_kernel.insert("layer " + std::to_string(il) + " " + role[k] + " " + it->second.dtype);+ }+ }+ }+ r.n = (int) ne;+ experts_bytes += (double) ne * (r.bytes[0] + r.bytes[1] + r.bytes[2]);+ }+ const double per_expert = nl > 0 ? experts_bytes / ((double) nl * (double) ne) : 0;+ // (a token picks `topk` experts a layer: sum over the layers of topk x one expert of that layer)+ double token_bytes = 0;+ for (long long il = 0; il < nl; ++il) token_bytes += (double) topk * (lay[(size_t) il].bytes[0] + lay[(size_t) il].bytes[1] + lay[(size_t) il].bytes[2]);+ const double expert_params = (double) nl * (double) ne * 3.0 * (double) ff * (double) dim;++ // ---- the dense side: everything that is not an expert tensor+ double dense = 0, dense_embed = 0, dense_head = 0, dense_mtp = 0, dense_vision = 0, dense_engram = 0;+ std::set<std::string> dense_types;+ for (const auto& [name, x] : t) {+ if (name.find(".ffn.experts.") != std::string::npos && name.rfind("layers.", 0) == 0) continue;+ dense += x.bytes;+ dense_types.insert(x.dtype);+ if (name == "embed.weight") dense_embed += x.bytes;+ else if (name == "head.weight") dense_head += x.bytes;+ else if (name.rfind("mtp.", 0) == 0) dense_mtp += x.bytes;+ else if (name.find(".engram.") != std::string::npos) dense_engram += x.bytes;+ else if (name.rfind("vision.", 0) == 0 || name.rfind("aligner.", 0) == 0) dense_vision += x.bytes;+ if (!dense_type(x.dtype)) no_kernel.insert("dense " + name + " " + x.dtype);+ }++ // ---- report+ std::fprintf(out, "pack-info %s\n", dir.c_str());+ std::fprintf(out, " model_dir %s format %s %lld layers x %lld experts, %lld used; dim %lld, ff %lld, vocab %lld\n",+ kv.count("model_dir") ? kv["model_dir"].c_str() : "?", kv.count("format") ? kv["format"].c_str() : "?", nl, ne, topk,+ dim, ff, vocab);+ std::fprintf(out, " layer gate+up down bytes/expert MB/token\n");+ for (long long il = 0; il < nl; ++il) {+ const Role& r = lay[(size_t) il];+ const double eb = r.bytes[0] + r.bytes[1] + r.bytes[2];+ std::fprintf(out, " %5lld %-9s %-9s %13.0f %10.2f%s\n", il, r.type[0].c_str(), r.type[2].c_str(), eb,+ (double) topk * eb / 1e6, r.mixed ? " (mixed)" : "");+ }+ std::fprintf(out, " expert tensors by type:");+ for (const auto& [ty, n] : type_count) std::fprintf(out, " %s %lld", ty.c_str(), n);+ std::fprintf(out, "\n");+ std::fprintf(out, " an expert %.2f MB, %lld picks a token: %.3f GB a token\n", per_expert / 1e6, topk, token_bytes / 1e9);+ std::fprintf(out, " all experts %.1f GB at %.2f bpw (%.0f G parameters), dense %.2f GB (embed %.2f, head %.2f, mtp %.2f, vision %.2f, engram %.2f)\n",+ gb(experts_bytes), expert_params > 0 ? experts_bytes * 8.0 / expert_params : 0.0, expert_params / 1e9, gb(dense),+ gb(dense_embed), gb(dense_head), gb(dense_mtp), gb(dense_vision), gb(dense_engram));+ std::fprintf(out, " dense types:");+ for (const auto& d : dense_types) std::fprintf(out, " %s", d.c_str());+ std::fprintf(out, "\n");+ if (no_kernel.empty()) {+ std::fprintf(out, " no GPU kernel: none (every tensor is a device type)\n");+ } else {+ std::fprintf(out, " no GPU kernel (%zu): the CPU lane takes them, they cannot go in the VRAM tier\n", no_kernel.size());+ int shown = 0;+ for (const std::string& s : no_kernel) {+ std::fprintf(out, " %s\n", s.c_str());+ if (++shown >= 40) {+ std::fprintf(out, " ... %zu more\n", no_kernel.size() - (size_t) shown);+ break;+ }+ }+ }+ return 0;+}++} // namespace strata::ds4diff --git a/src/dsv41/ds4_packinfo.hpp b/src/dsv41/ds4_packinfo.hppnew file mode 100644index 000000000..c7597eebf--- /dev/null+++ b/src/dsv41/ds4_packinfo.hpp@@ -0,0 +1,13 @@+// Strata. The pack report (ds4_packinfo.cpp): `strata-dsv41 --pack-info --pack <dir>`. Host only: no device, no weights.+#pragma once++#include <cstdio>+#include <string>++namespace strata::ds4 {++/// Prints the pack's type map, its bytes an expert / a token, its tier sizes and the tensors whose type has no device+/// kernel. 0 when the pack's description was read, non-zero (and a line saying what is missing) otherwise.+int pack_info(const std::string& dir, std::FILE* out);++} // namespace strata::ds4