Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek Metal: sampler statistics harness, round time breakdown
Niko1221958774b4ceb6
diff --git a/src/dsv41/ds4_main.cpp b/src/dsv41/ds4_main.cppindex 283f43639..bd7c85847 100644--- a/src/dsv41/ds4_main.cpp+++ b/src/dsv41/ds4_main.cpp@@ -143,6 +143,17 @@ static int g_round_nd = 0;// One round of decoding after `tok` (the token at pos(), not fed yet): with the drafter, its drafts while their// confidence holds (at most `room` - 1 of them), verified together; else a plain step. `out` gets the tokens that follow// tok (the accepted drafts, then the trunk's next), the last of them the next round's tok.+struct RoundTimes { // DS4_ROUND_TIMES: where a decode round's time goes (ms, summed), printed at exit+ double draft = 0, verify = 0, step = 0;+ long rounds = 0, spec_rounds = 0, steps = 0;+ ~RoundTimes() {+ if (std::getenv("DS4_ROUND_TIMES") && rounds > 0)+ std::fprintf(stderr, "rounds: %ld (%ld speculative, %ld plain steps): draft %.1f ms (%.2f a round), verify %.1f ms (%.2f), plain step %.1f ms (%.2f)\n",+ rounds, spec_rounds, steps, draft, spec_rounds ? draft / (double) spec_rounds : 0.0, verify, spec_rounds ? verify / (double) spec_rounds : 0.0,+ step, steps ? step / (double) steps : 0.0);+ }+} g_rt;+static double rt_now() { return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now().time_since_epoch()).count(); }bool round(strata::ds4::Model& m, int tok, int room, bool spec, float conf_min, float temperature, float top_p, int top_k,float min_p, unsigned long long seed, unsigned long long& counter, std::vector<int>& out, int& offered,int& accepted, std::string& err) {@@ -151,7 +162,10 @@ bool round(strata::ds4::Model& m, int tok, int room, bool spec, float conf_min,if (spec && room > 1 && m.draft_block() > 0) {std::vector<int> drafts;std::vector<float> conf;+ ++g_rt.rounds;+ const double rt0 = rt_now();if (!m.draft(tok, drafts, conf, err, temperature, top_p, top_k, min_p, seed, counter)) return false;+ g_rt.draft += rt_now() - rt0;g_round_conf = conf;g_round_nd = (int) drafts.size();int keep = 0;@@ -159,7 +173,10 @@ bool round(strata::ds4::Model& m, int tok, int room, bool spec, float conf_min,if (keep > 0) {drafts.resize((size_t) keep);int acc = 0, next = -1;+ const double rt1 = rt_now();if (!m.verify(tok, drafts, temperature, top_p, top_k, min_p, seed, counter, acc, next, err)) return false;+ g_rt.verify += rt_now() - rt1;+ ++g_rt.spec_rounds;if (g_round_log != nullptr) { // DS4_ROUND_LOG: tag, drafts offered, accepted, the confidencesstd::fprintf(g_round_log, "%s\t%d\t%d", g_round_tag.c_str(), keep, acc);for (int i = 0; i < keep; ++i) std::fprintf(g_round_log, "\t%.4f", g_round_conf[(size_t) i]);@@ -173,8 +190,11 @@ bool round(strata::ds4::Model& m, int tok, int room, bool spec, float conf_min,return true;}}+ const double rt2 = rt_now();if (!m.step(tok, true, err)) return false;out.push_back(m.sample(temperature, top_p, top_k, min_p, seed, counter++));+ g_rt.step += rt_now() - rt2;+ ++g_rt.steps;return true;}@@ -738,7 +758,13 @@ int spec_eval(strata::ds4::Model& m, const std::string& ids_path, int n, int win}} // namespace+#if defined(STRATA_METAL)+#include "sampler_stat.inc"+#endifint main(int argc, char** argv) {+#if defined(STRATA_METAL)+ if (argc > 1 && std::string(argv[1]) == "--sampler-test") return sampler_stat::run(argc - 1, argv + 1); // (the sampler's statistics harness)+#endif#if defined(__APPLE__)if (std::getenv("DS4_BT")) { // a crash prints where it was (no debugger on the Mac box)signal(SIGSEGV, [](int) { void* fr[40]; const int n = backtrace(fr, 40); backtrace_symbols_fd(fr, n, 2); _exit(139); });diff --git a/src/dsv41/sampler_stat.inc b/src/dsv41/sampler_stat.incnew file mode 100644index 000000000..01d2a4d0f--- /dev/null+++ b/src/dsv41/sampler_stat.inc@@ -0,0 +1,107 @@+// sampler_stat.inc - a statistics harness for the sampler (sample_tokens / sample_draft / sample_verify): the same text is compiled+// into the Metal engine (--sampler-test) and into a CUDA standalone, so the host sampler (Metal) and the CUDA kernel can be compared+// sample by sample (same Philox counters -> identical picks when the arithmetic agrees) and against the chain's own distribution.+// sampler_stat ROWS.bin N_VOCAB N_SAMPLES OUTPREFIX ROWS.bin: 4 float32 rows of N_VOCAB: peaked, flat, penalty case, draft+// Writes OUTPREFIX_A.bin (peaked, top_k 20 top_p 0.95 T 0.8), _B.bin (flat, top_k 64, T 1), _C.bin (penalties on a 64-token history,+// top_k 20 top_p 0.9 T 0.7), _D.bin (speculative: the draft tokens, then the verify's outputs, int32 each).+#include <algorithm>+#include <cstdint>+#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <string>+#include <vector>++#include <cuda_runtime.h>+#include "strata/dsv41/sampler.hpp"++namespace sampler_stat {+using strata::dsv41k::SamplerParams;+using strata::dsv41k::SpecList;++inline int run(int argc, char** argv) {+ if (argc < 5) {+ std::fprintf(stderr, "usage: sampler_stat ROWS.bin N_VOCAB N_SAMPLES OUTPREFIX\n");+ return 2;+ }+ const int V = std::atoi(argv[2]), N = std::atoi(argv[3]);+ const std::string pre = argv[4];+ std::vector<float> rows((size_t) 4 * V);+ {+ std::FILE* f = std::fopen(argv[1], "rb");+ if (f == nullptr || std::fread(rows.data(), 4, rows.size(), f) != rows.size()) {+ std::fprintf(stderr, "cannot read %s\n", argv[1]);+ return 2;+ }+ std::fclose(f);+ }+ const int B = 500; // rows a launch+ float* dl = nullptr;+ float* dq = nullptr;+ int* dout = nullptr;+ int* dd = nullptr;+ int* dhist = nullptr;+ SpecList* dql = nullptr;+ cudaMalloc((void**) &dl, (size_t) B * V * 4);+ cudaMalloc((void**) &dq, (size_t) B * V * 4);+ cudaMalloc((void**) &dout, (size_t) B * 4);+ cudaMalloc((void**) &dd, (size_t) B * 4);+ cudaMalloc((void**) &dql, sizeof(SpecList) * (size_t) B);+ std::vector<int> hist(64);+ for (int i = 0; i < 64; ++i) hist[(size_t) i] = (int) (((long long) i * 37 + 11) % V);+ // the history is per row (history_len ints a row): the same 64 for every row of a launch+ std::vector<int> hist_rows((size_t) B * 64);+ for (int r = 0; r < B; ++r)+ for (int i = 0; i < 64; ++i) hist_rows[(size_t) r * 64 + (size_t) i] = hist[(size_t) i];+ cudaMalloc((void**) &dhist, hist_rows.size() * 4);+ cudaMemcpy(dhist, hist_rows.data(), hist_rows.size() * 4, cudaMemcpyHostToDevice);++ auto fill = [&](float* dev, const float* row) {+ std::vector<float> m((size_t) B * V);+ for (int r = 0; r < B; ++r) std::memcpy(m.data() + (size_t) r * V, row, (size_t) V * 4);+ cudaMemcpy(dev, m.data(), m.size() * 4, cudaMemcpyHostToDevice);+ };+ auto write = [&](const char* tag, const std::vector<int>& v) {+ std::FILE* f = std::fopen((pre + "_" + tag + ".bin").c_str(), "wb");+ std::fwrite(v.data(), 4, v.size(), f);+ std::fclose(f);+ };+ std::vector<int> res(B);+ for (int c = 0; c < 4; ++c) {+ const float* row = rows.data() + (size_t) (c == 3 ? 0 : c) * V;+ SamplerParams p;+ p.seed = 12345;+ const char* tag = "A";+ if (c == 0) { p.top_k = 20; p.top_p = 0.95f; p.temperature = 0.8f; }+ if (c == 1) { p.top_k = 0; p.top_p = 1.0f; p.temperature = 1.0f; tag = "B"; }+ if (c == 2) {+ p.top_k = 20; p.top_p = 0.9f; p.temperature = 0.7f; tag = "C";+ p.penalty_last_n = 64; p.penalty_repeat = 1.3f; p.penalty_freq = 0.1f; p.penalty_present = 0.1f;+ }+ if (c == 3) { p.top_k = 20; p.top_p = 0.95f; p.temperature = 0.8f; tag = "D"; }+ fill(dl, row);+ if (c == 3) fill(dq, rows.data() + (size_t) 3 * V);+ std::vector<int> all, all_d;+ for (int base = 0; base < N; base += B) {+ const int n = std::min(B, N - base);+ p.counter = (uint64_t) base;+ if (c == 3) {+ strata::dsv41k::sample_draft(dq, n, V, p, dql, dd, nullptr);+ strata::dsv41k::sample_verify(dl, n, V, p, dql, dd, dout, nullptr);+ cudaDeviceSynchronize();+ cudaMemcpy(res.data(), dd, (size_t) n * 4, cudaMemcpyDeviceToHost);+ all_d.insert(all_d.end(), res.begin(), res.begin() + n);+ } else {+ strata::dsv41k::sample_tokens(dl, n, V, c == 2 ? dhist : nullptr, c == 2 ? 64 : 0, p, dout, nullptr);+ cudaDeviceSynchronize();+ }+ cudaMemcpy(res.data(), dout, (size_t) n * 4, cudaMemcpyDeviceToHost);+ all.insert(all.end(), res.begin(), res.begin() + n);+ }+ if (c == 3) all.insert(all.begin(), all_d.begin(), all_d.end()); // drafts first, then the verify's outputs+ write(tag, all);+ std::fprintf(stderr, "sampler_stat: case %s done (%zu values)\n", tag, all.size());+ }+ return 0;+}+} // namespace sampler_statdiff --git a/src/dsv41/sampler_stat_main.cu b/src/dsv41/sampler_stat_main.cunew file mode 100644index 000000000..878ad06cc--- /dev/null+++ b/src/dsv41/sampler_stat_main.cu@@ -0,0 +1,4 @@+// CUDA standalone of the sampler statistics harness: nvcc -include ds4_launch.h -I include -I src/dsv41 sampler_stat_main.cu+#include "sampler_stat.inc"+#include "vendor/sampler.cu"+int main(int argc, char** argv) { return sampler_stat::run(argc, argv); }