Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

DeepSeek Metal: sampler statistics harness, round time breakdown

Niko1221958774b4ceb6
diff --git a/src/dsv41/ds4_main.cpp b/src/dsv41/ds4_main.cpp
index 283f43639..bd7c85847 100644
--- a/src/dsv41/ds4_main.cpp
+++ b/src/dsv41/ds4_main.cpp
@@ -143,6 +143,17 @@ static int g_round_nd = 0;
// One round of decoding after `tok` (the token at pos(), not fed yet): with the drafter, its drafts while their
// confidence holds (at most `room` - 1 of them), verified together; else a plain step. `out` gets the tokens that follow
// tok (the accepted drafts, then the trunk's next), the last of them the next round's tok.
+struct RoundTimes { // DS4_ROUND_TIMES: where a decode round's time goes (ms, summed), printed at exit
+ double draft = 0, verify = 0, step = 0;
+ long rounds = 0, spec_rounds = 0, steps = 0;
+ ~RoundTimes() {
+ if (std::getenv("DS4_ROUND_TIMES") && rounds > 0)
+ std::fprintf(stderr, "rounds: %ld (%ld speculative, %ld plain steps): draft %.1f ms (%.2f a round), verify %.1f ms (%.2f), plain step %.1f ms (%.2f)\n",
+ rounds, spec_rounds, steps, draft, spec_rounds ? draft / (double) spec_rounds : 0.0, verify, spec_rounds ? verify / (double) spec_rounds : 0.0,
+ step, steps ? step / (double) steps : 0.0);
+ }
+} g_rt;
+static double rt_now() { return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now().time_since_epoch()).count(); }
bool round(strata::ds4::Model& m, int tok, int room, bool spec, float conf_min, float temperature, float top_p, int top_k,
float min_p, unsigned long long seed, unsigned long long& counter, std::vector<int>& out, int& offered,
int& accepted, std::string& err) {
@@ -151,7 +162,10 @@ bool round(strata::ds4::Model& m, int tok, int room, bool spec, float conf_min,
if (spec && room > 1 && m.draft_block() > 0) {
std::vector<int> drafts;
std::vector<float> conf;
+ ++g_rt.rounds;
+ const double rt0 = rt_now();
if (!m.draft(tok, drafts, conf, err, temperature, top_p, top_k, min_p, seed, counter)) return false;
+ g_rt.draft += rt_now() - rt0;
g_round_conf = conf;
g_round_nd = (int) drafts.size();
int keep = 0;
@@ -159,7 +173,10 @@ bool round(strata::ds4::Model& m, int tok, int room, bool spec, float conf_min,
if (keep > 0) {
drafts.resize((size_t) keep);
int acc = 0, next = -1;
+ const double rt1 = rt_now();
if (!m.verify(tok, drafts, temperature, top_p, top_k, min_p, seed, counter, acc, next, err)) return false;
+ g_rt.verify += rt_now() - rt1;
+ ++g_rt.spec_rounds;
if (g_round_log != nullptr) { // DS4_ROUND_LOG: tag, drafts offered, accepted, the confidences
std::fprintf(g_round_log, "%s\t%d\t%d", g_round_tag.c_str(), keep, acc);
for (int i = 0; i < keep; ++i) std::fprintf(g_round_log, "\t%.4f", g_round_conf[(size_t) i]);
@@ -173,8 +190,11 @@ bool round(strata::ds4::Model& m, int tok, int room, bool spec, float conf_min,
return true;
}
}
+ const double rt2 = rt_now();
if (!m.step(tok, true, err)) return false;
out.push_back(m.sample(temperature, top_p, top_k, min_p, seed, counter++));
+ g_rt.step += rt_now() - rt2;
+ ++g_rt.steps;
return true;
}
@@ -738,7 +758,13 @@ int spec_eval(strata::ds4::Model& m, const std::string& ids_path, int n, int win
}
} // namespace
+#if defined(STRATA_METAL)
+#include "sampler_stat.inc"
+#endif
int main(int argc, char** argv) {
+#if defined(STRATA_METAL)
+ if (argc > 1 && std::string(argv[1]) == "--sampler-test") return sampler_stat::run(argc - 1, argv + 1); // (the sampler's statistics harness)
+#endif
#if defined(__APPLE__)
if (std::getenv("DS4_BT")) { // a crash prints where it was (no debugger on the Mac box)
signal(SIGSEGV, [](int) { void* fr[40]; const int n = backtrace(fr, 40); backtrace_symbols_fd(fr, n, 2); _exit(139); });
diff --git a/src/dsv41/sampler_stat.inc b/src/dsv41/sampler_stat.inc
new file mode 100644
index 000000000..01d2a4d0f
--- /dev/null
+++ b/src/dsv41/sampler_stat.inc
@@ -0,0 +1,107 @@
+// sampler_stat.inc - a statistics harness for the sampler (sample_tokens / sample_draft / sample_verify): the same text is compiled
+// into the Metal engine (--sampler-test) and into a CUDA standalone, so the host sampler (Metal) and the CUDA kernel can be compared
+// sample by sample (same Philox counters -> identical picks when the arithmetic agrees) and against the chain's own distribution.
+// sampler_stat ROWS.bin N_VOCAB N_SAMPLES OUTPREFIX ROWS.bin: 4 float32 rows of N_VOCAB: peaked, flat, penalty case, draft
+// Writes OUTPREFIX_A.bin (peaked, top_k 20 top_p 0.95 T 0.8), _B.bin (flat, top_k 64, T 1), _C.bin (penalties on a 64-token history,
+// top_k 20 top_p 0.9 T 0.7), _D.bin (speculative: the draft tokens, then the verify's outputs, int32 each).
+#include <algorithm>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <string>
+#include <vector>
+
+#include <cuda_runtime.h>
+#include "strata/dsv41/sampler.hpp"
+
+namespace sampler_stat {
+using strata::dsv41k::SamplerParams;
+using strata::dsv41k::SpecList;
+
+inline int run(int argc, char** argv) {
+ if (argc < 5) {
+ std::fprintf(stderr, "usage: sampler_stat ROWS.bin N_VOCAB N_SAMPLES OUTPREFIX\n");
+ return 2;
+ }
+ const int V = std::atoi(argv[2]), N = std::atoi(argv[3]);
+ const std::string pre = argv[4];
+ std::vector<float> rows((size_t) 4 * V);
+ {
+ std::FILE* f = std::fopen(argv[1], "rb");
+ if (f == nullptr || std::fread(rows.data(), 4, rows.size(), f) != rows.size()) {
+ std::fprintf(stderr, "cannot read %s\n", argv[1]);
+ return 2;
+ }
+ std::fclose(f);
+ }
+ const int B = 500; // rows a launch
+ float* dl = nullptr;
+ float* dq = nullptr;
+ int* dout = nullptr;
+ int* dd = nullptr;
+ int* dhist = nullptr;
+ SpecList* dql = nullptr;
+ cudaMalloc((void**) &dl, (size_t) B * V * 4);
+ cudaMalloc((void**) &dq, (size_t) B * V * 4);
+ cudaMalloc((void**) &dout, (size_t) B * 4);
+ cudaMalloc((void**) &dd, (size_t) B * 4);
+ cudaMalloc((void**) &dql, sizeof(SpecList) * (size_t) B);
+ std::vector<int> hist(64);
+ for (int i = 0; i < 64; ++i) hist[(size_t) i] = (int) (((long long) i * 37 + 11) % V);
+ // the history is per row (history_len ints a row): the same 64 for every row of a launch
+ std::vector<int> hist_rows((size_t) B * 64);
+ for (int r = 0; r < B; ++r)
+ for (int i = 0; i < 64; ++i) hist_rows[(size_t) r * 64 + (size_t) i] = hist[(size_t) i];
+ cudaMalloc((void**) &dhist, hist_rows.size() * 4);
+ cudaMemcpy(dhist, hist_rows.data(), hist_rows.size() * 4, cudaMemcpyHostToDevice);
+
+ auto fill = [&](float* dev, const float* row) {
+ std::vector<float> m((size_t) B * V);
+ for (int r = 0; r < B; ++r) std::memcpy(m.data() + (size_t) r * V, row, (size_t) V * 4);
+ cudaMemcpy(dev, m.data(), m.size() * 4, cudaMemcpyHostToDevice);
+ };
+ auto write = [&](const char* tag, const std::vector<int>& v) {
+ std::FILE* f = std::fopen((pre + "_" + tag + ".bin").c_str(), "wb");
+ std::fwrite(v.data(), 4, v.size(), f);
+ std::fclose(f);
+ };
+ std::vector<int> res(B);
+ for (int c = 0; c < 4; ++c) {
+ const float* row = rows.data() + (size_t) (c == 3 ? 0 : c) * V;
+ SamplerParams p;
+ p.seed = 12345;
+ const char* tag = "A";
+ if (c == 0) { p.top_k = 20; p.top_p = 0.95f; p.temperature = 0.8f; }
+ if (c == 1) { p.top_k = 0; p.top_p = 1.0f; p.temperature = 1.0f; tag = "B"; }
+ if (c == 2) {
+ p.top_k = 20; p.top_p = 0.9f; p.temperature = 0.7f; tag = "C";
+ p.penalty_last_n = 64; p.penalty_repeat = 1.3f; p.penalty_freq = 0.1f; p.penalty_present = 0.1f;
+ }
+ if (c == 3) { p.top_k = 20; p.top_p = 0.95f; p.temperature = 0.8f; tag = "D"; }
+ fill(dl, row);
+ if (c == 3) fill(dq, rows.data() + (size_t) 3 * V);
+ std::vector<int> all, all_d;
+ for (int base = 0; base < N; base += B) {
+ const int n = std::min(B, N - base);
+ p.counter = (uint64_t) base;
+ if (c == 3) {
+ strata::dsv41k::sample_draft(dq, n, V, p, dql, dd, nullptr);
+ strata::dsv41k::sample_verify(dl, n, V, p, dql, dd, dout, nullptr);
+ cudaDeviceSynchronize();
+ cudaMemcpy(res.data(), dd, (size_t) n * 4, cudaMemcpyDeviceToHost);
+ all_d.insert(all_d.end(), res.begin(), res.begin() + n);
+ } else {
+ strata::dsv41k::sample_tokens(dl, n, V, c == 2 ? dhist : nullptr, c == 2 ? 64 : 0, p, dout, nullptr);
+ cudaDeviceSynchronize();
+ }
+ cudaMemcpy(res.data(), dout, (size_t) n * 4, cudaMemcpyDeviceToHost);
+ all.insert(all.end(), res.begin(), res.begin() + n);
+ }
+ if (c == 3) all.insert(all.begin(), all_d.begin(), all_d.end()); // drafts first, then the verify's outputs
+ write(tag, all);
+ std::fprintf(stderr, "sampler_stat: case %s done (%zu values)\n", tag, all.size());
+ }
+ return 0;
+}
+} // namespace sampler_stat
diff --git a/src/dsv41/sampler_stat_main.cu b/src/dsv41/sampler_stat_main.cu
new file mode 100644
index 000000000..878ad06cc
--- /dev/null
+++ b/src/dsv41/sampler_stat_main.cu
@@ -0,0 +1,4 @@
+// CUDA standalone of the sampler statistics harness: nvcc -include ds4_launch.h -I include -I src/dsv41 sampler_stat_main.cu
+#include "sampler_stat.inc"
+#include "vendor/sampler.cu"
+int main(int argc, char** argv) { return sampler_stat::run(argc, argv); }