Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

DeepSeek Metal: drafter bring-up (ring restore, host sampler for temperature and speculative sampling), per-dispatch GPU timeline

Niko1221f29ff5a59a10
diff --git a/metal/src/mtl_rt.mm b/metal/src/mtl_rt.mm
index 93c75be27..12816ab0d 100644
--- a/metal/src/mtl_rt.mm
+++ b/metal/src/mtl_rt.mm
@@ -13,6 +13,7 @@
#include <unordered_map>
#include <string>
#include <deque>
+#include <mach/mach_time.h>
#include <map>
#include <thread>
#include <cstdio>
@@ -66,8 +67,150 @@ static inline id<MTLBuffer> BO(const Buffer& b, size_t off, size_t& o) {
static inline id<MTLComputePipelineState> P(const Pipeline* p) { return (__bridge id<MTLComputePipelineState>)p->handle; }
static std::string g_device_name = "unknown GPU";
+
+// ---- DS4_TIMELINE=1: every deferred dispatch in its own encoder with start / end GPU timestamps; the report at exit gives, per kernel,
+// its GPU time and the gap the GPU sat idle before it (host waits, event waits, launch). DS4_TL_FROM=N: only from dispatch N on.
+static std::atomic<uint64_t> g_disp_count{0}; // dispatches encoded
+namespace {
+struct Timeline {
+ bool on = false;
+ id<MTLCounterSet> tset = nil;
+ std::vector<id<MTLCounterSampleBuffer>> bufs; // a sample buffer holds only ~1500 encoders: they rotate
+ size_t per = 0, cap = 0, n = 0; // encoders per buffer / in all (the buffers' limit) / recorded
+ long after = 0; // DS4_TL_AFTER: record from the Nth dispatch on
+ std::vector<std::string> names; // per encoder
+ uint64_t cpu0 = 0, gpu0 = 0;
+ bool reg = false;
+ bool att = false; // the encoder just made carries a sample slot
+} g_tl;
+
+id<MTLComputeCommandEncoder> tl_make_encoder(id<MTLDevice> dev, id<MTLCommandBuffer> cb, MTLDispatchType dt) {
+ g_tl.att = false;
+ if (!g_tl.on) return [cb computeCommandEncoderWithDispatchType:dt];
+ if (g_tl.tset == nil) {
+ for (id<MTLCounterSet> cs in dev.counterSets)
+ if ([cs.name isEqualToString:MTLCommonCounterSetTimestamp]) g_tl.tset = cs;
+ if (g_tl.tset == nil) {
+ std::fprintf(stderr, "mtl: DS4_TIMELINE: no timestamp counter set\n");
+ g_tl.on = false;
+ return [cb computeCommandEncoderWithDispatchType:dt];
+ }
+ for (size_t c = 4096; c >= 64 && g_tl.per == 0; c /= 2) {
+ MTLCounterSampleBufferDescriptor* d = [MTLCounterSampleBufferDescriptor new];
+ d.counterSet = g_tl.tset;
+ d.storageMode = MTLStorageModeShared;
+ d.sampleCount = c * 2;
+ NSError* err = nil;
+ id<MTLCounterSampleBuffer> b = [dev newCounterSampleBufferWithDescriptor:d error:&err];
+ if (b != nil) {
+ g_tl.per = c;
+ g_tl.bufs.push_back(b);
+ }
+ }
+ if (g_tl.per == 0) {
+ std::fprintf(stderr, "mtl: DS4_TIMELINE: no counter sample buffer\n");
+ g_tl.on = false;
+ return [cb computeCommandEncoderWithDispatchType:dt];
+ }
+ g_tl.cap = g_tl.per * 120;
+ MTLTimestamp c0 = 0, g0 = 0;
+ [dev sampleTimestamps:&c0 gpuTimestamp:&g0];
+ g_tl.cpu0 = c0;
+ g_tl.gpu0 = g0;
+ g_tl.names.assign(g_tl.cap, std::string());
+ g_tl.after = std::getenv("DS4_TL_AFTER") ? std::atol(std::getenv("DS4_TL_AFTER")) : 0;
+ }
+ if (g_tl.n >= g_tl.cap || (long) g_disp_count.load() < g_tl.after) return [cb computeCommandEncoderWithDispatchType:dt];
+ const size_t bi = g_tl.n / g_tl.per, si = g_tl.n % g_tl.per;
+ if (bi >= g_tl.bufs.size()) {
+ MTLCounterSampleBufferDescriptor* d = [MTLCounterSampleBufferDescriptor new];
+ d.counterSet = g_tl.tset;
+ d.storageMode = MTLStorageModeShared;
+ d.sampleCount = g_tl.per * 2;
+ NSError* err = nil;
+ id<MTLCounterSampleBuffer> b = [dev newCounterSampleBufferWithDescriptor:d error:&err];
+ if (b == nil) {
+ g_tl.cap = g_tl.n;
+ return [cb computeCommandEncoderWithDispatchType:dt];
+ }
+ g_tl.bufs.push_back(b);
+ }
+ MTLComputePassDescriptor* pd = [MTLComputePassDescriptor computePassDescriptor];
+ pd.dispatchType = dt;
+ pd.sampleBufferAttachments[0].sampleBuffer = g_tl.bufs[bi];
+ pd.sampleBufferAttachments[0].startOfEncoderSampleIndex = 2 * si;
+ pd.sampleBufferAttachments[0].endOfEncoderSampleIndex = 2 * si + 1;
+ g_tl.att = true;
+ return [cb computeCommandEncoderWithDescriptor:pd];
+}
+
+void tl_report(id<MTLDevice> dev) {
+ if (!g_tl.on && g_tl.bufs.empty()) return;
+ const size_t n = std::min(g_tl.n, g_tl.cap);
+ if (n < 2) return;
+ std::vector<MTLCounterResultTimestamp> tsv(2 * n);
+ for (size_t b = 0; b * g_tl.per < n; ++b) {
+ const size_t cnt = std::min(g_tl.per, n - b * g_tl.per);
+ NSData* data = [g_tl.bufs[b] resolveCounterRange:NSMakeRange(0, 2 * cnt)];
+ if (data == nil) { std::fprintf(stderr, "mtl: DS4_TIMELINE: resolve failed\n"); return; }
+ std::memcpy(&tsv[2 * b * g_tl.per], [data bytes], 2 * cnt * sizeof(MTLCounterResultTimestamp));
+ }
+ const MTLCounterResultTimestamp* ts = tsv.data();
+ MTLTimestamp c1 = 0, g1 = 0;
+ [dev sampleTimestamps:&c1 gpuTimestamp:&g1];
+ mach_timebase_info_data_t tb;
+ mach_timebase_info(&tb);
+ const double cpu_ns_per_tick = (double) tb.numer / (double) tb.denom;
+ (void) cpu_ns_per_tick; // (sampleTimestamps' CPU value is in ns already)
+ const double ns_per_gpu = (g1 > g_tl.gpu0) ? (double) (c1 - g_tl.cpu0) / (double) (g1 - g_tl.gpu0) : 1.0;
+ std::fprintf(stderr, "timeline: cap %zu n %zu scale %.6f ns/tick first ts %llu %llu last %llu\n", g_tl.cap, g_tl.n, ns_per_gpu, (unsigned long long) ts[0].timestamp, (unsigned long long) ts[1].timestamp, (unsigned long long) ts[2 * n - 1].timestamp);
+ size_t from = std::getenv("DS4_TL_FROM") ? (size_t) std::atoll(std::getenv("DS4_TL_FROM")) : 0;
+ struct Acc { double busy = 0, gap = 0, gap_big = 0; size_t n = 0, n_big = 0; };
+ std::map<std::string, Acc> acc;
+ double t_first = 0, t_last = 0, total_busy = 0, total_gap = 0, total_big = 0;
+ double prev_end = -1;
+ size_t used = 0;
+ for (size_t k = from; k < n; ++k) {
+ if (g_tl.names[k].empty()) continue;
+ const double st = (double) ts[2 * k].timestamp * ns_per_gpu * 1e-3, en = (double) ts[2 * k + 1].timestamp * ns_per_gpu * 1e-3; // us
+ if (!(en >= st)) continue;
+ if (prev_end < 0) t_first = st;
+ Acc& a = acc[g_tl.names[k]];
+ a.busy += en - st;
+ ++a.n;
+ total_busy += en - st;
+ if (prev_end >= 0 && st >= prev_end) {
+ const double gp = st - prev_end;
+ a.gap += gp;
+ total_gap += gp;
+ if (gp > 300.0) { a.gap_big += gp; ++a.n_big; total_big += gp; }
+ }
+ prev_end = en;
+ t_last = en;
+ ++used;
+ }
+ std::vector<std::pair<double, std::string>> v;
+ for (auto& kv : acc) v.emplace_back(kv.second.busy + kv.second.gap, kv.first);
+ std::sort(v.begin(), v.end(), [](auto& x, auto& y) { return x.first > y.first; });
+ std::fprintf(stderr, "timeline: %zu dispatches, wall %.1f ms, GPU busy %.1f ms, idle gaps %.1f ms (of it > 300 us: %.1f ms)\n", used,
+ (t_last - t_first) / 1000.0, total_busy / 1000.0, total_gap / 1000.0, total_big / 1000.0);
+ std::fprintf(stderr, "timeline: kernel n busy ms us/disp gap ms gap/disp big-gap ms (n)\n");
+ for (size_t i = 0; i < v.size() && i < 40; ++i) {
+ const Acc& a = acc[v[i].second];
+ std::fprintf(stderr, "timeline: %-28s %6zu %9.1f %8.1f %8.1f %8.1f %10.1f (%zu)\n", v[i].second.c_str(), a.n, a.busy / 1000.0, a.busy / (double) a.n,
+ a.gap / 1000.0, a.gap / (double) a.n, a.gap_big / 1000.0, a.n_big);
+ }
+}
+} // namespace
+
Context::Context() : impl_(new Impl) {
impl_->dev = MTLCreateSystemDefaultDevice();
+ if (std::getenv("DS4_TIMELINE") && !g_tl.reg) {
+ g_tl.on = true;
+ g_tl.reg = true;
+ static id<MTLDevice> tl_dev = impl_->dev;
+ std::atexit([] { tl_report(tl_dev); });
+ }
if (!impl_->dev) {
std::fprintf(stderr, "mtl: no Metal device\n");
std::exit(2);
@@ -420,7 +563,6 @@ static std::atomic<uint64_t> g_wait_ns{0}, g_enc_ns{0}, g_wait_n{0}; // host t
static std::mutex g_prof_mu;
static std::unordered_map<std::string, std::pair<double, uint64_t>> g_prof; // DS4_PROF: kernel -> (GPU us, dispatches)
static std::string g_prof_last;
-static std::atomic<uint64_t> g_disp_count{0}; // dispatches encoded, and command buffers committed
static std::atomic<uint64_t> g_cb_count{0};
static bool g_count_reg = false;
static void ds4_report_counts() {
@@ -551,10 +693,11 @@ double Context::run(const std::vector<Dispatch>& ds, int reps, bool concurrent,
} else
impl_->cb_def = [impl_->q commandBufferWithUnretainedReferences];
if (impl_->qev != nil && impl_->qev_v > 0) [impl_->cb_def encodeWaitForEvent:impl_->qev value:impl_->qev_v];
- impl_->enc_def = [impl_->cb_def computeCommandEncoderWithDispatchType:std::getenv("DS4_CONCURRENT") ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial];
+ impl_->enc_def = tl_make_encoder(impl_->dev, impl_->cb_def, std::getenv("DS4_CONCURRENT") ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial);
}
const auto te0 = std::chrono::steady_clock::now();
encode(impl_->enc_def, ds[0]);
+ if (g_tl.on && g_tl.att && g_tl.n < g_tl.cap) { g_tl.names[g_tl.n++] = ds[0].p->name; g_tl.att = false; }
g_last_act_ns.store(ka_now_ns(), std::memory_order_relaxed);
g_recent.push_back(ds[0].p->name); if (g_recent.size() > 8) g_recent.pop_front();
if (std::getenv("DS4_ERRINFO")) impl_->def_names += ds[0].p->name + " ";
@@ -566,10 +709,10 @@ double Context::run(const std::vector<Dispatch>& ds, int reps, bool concurrent,
// second matvec's activation quantiser read the previous layer's x).
static const bool no_mb = std::getenv("DS4_NO_MBARRIER") != nullptr;
if (!no_mb) [impl_->enc_def memoryBarrierWithScope:MTLBarrierScopeBuffers];
- static const bool enc_each = std::getenv("DS4_ENC_EACH") != nullptr; // a new encoder per dispatch (debugging)
+ static const bool enc_each = std::getenv("DS4_ENC_EACH") != nullptr || g_tl.on; // a new encoder per dispatch (debugging, the timeline)
if (enc_each) {
[impl_->enc_def endEncoding];
- impl_->enc_def = [impl_->cb_def computeCommandEncoderWithDispatchType:(std::getenv("DS4_CONCURRENT") ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial)];
+ impl_->enc_def = tl_make_encoder(impl_->dev, impl_->cb_def, (std::getenv("DS4_CONCURRENT") ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial));
}
++impl_->def_n;
++g_disp_count;
diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cu
index f776f88fc..f7edf5745 100644
--- a/src/dsv41/ds4_kernels.cu
+++ b/src/dsv41/ds4_kernels.cu
@@ -2196,10 +2196,26 @@ void ext_to_ring(const uint16_t* ext, uint8_t* ring, int64_t p0, int n, int T, i
strata_ds4_launch(ext_to_ring_kernel, T, kThreads, 0, s, ext, ring, (long long) p0, n, T, d, backup);
}
void ring_restore(uint8_t* ring, const uint8_t* backup, int64_t p0, int n, int keep, int T, int d, cudaStream_t s) {
+#if defined(STRATA_METAL)
+ { // (row copies on the context queue: the drafter's rollback is a few rows)
+ const size_t rb = kv_row_bytes(kKvFp8, d);
+ for (int t = keep; t < T; ++t) cudaMemcpyAsync(ring + (size_t) ((p0 + t) % n) * rb, backup + (size_t) t * rb, rb, cudaMemcpyDeviceToDevice, s);
+ return;
+ }
+#endif
if (T > keep)
strata_ds4_launch(ring_restore_kernel, T - keep, kThreads, 0, s, ring, backup, (long long) p0, n, keep, kv_row_bytes(kKvFp8, d));
}
void ring_restore_batch(const RingBatch& b, int64_t p0, int n, int keep, int T, int d, cudaStream_t s) {
+#if defined(STRATA_METAL)
+ {
+ const size_t rb = kv_row_bytes(kKvFp8, d);
+ for (int li = 0; li < b.n; ++li)
+ for (int t = keep; t < T; ++t)
+ cudaMemcpyAsync(b.ring[li] + (size_t) ((p0 + t) % n) * rb, b.bak[li] + (size_t) t * rb, rb, cudaMemcpyDeviceToDevice, s);
+ return;
+ }
+#endif
if (T > keep && b.n > 0)
strata_ds4_launch(ring_restore_batch_kernel, dim3(T - keep, b.n), kThreads, 0, s, b, (long long) p0, n, keep,
kv_row_bytes(kKvFp8, d));
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index b458494f6..98d008f8b 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -8391,6 +8391,7 @@ struct Model::Impl {
#endif
static const bool skip_cpu = std::getenv("DS4_SKIP_CPU") != nullptr; // (timing only: the CPU lane's experts left out)
if (!skip_cpu) cpu_experts(l, h_x, ents);
+ else if (const char* fk = std::getenv("DS4_FAKE_CPU_US")) std::this_thread::sleep_for(std::chrono::microseconds(std::atoi(fk))); // (timing only)
std::fill(h_part, h_part + (size_t) T * D, 0.0f);
for (int sl = 0; sl < nslots; ++sl) {
float* o = h_part + (size_t) cpu_rows[(size_t) sl].first * D;
diff --git a/src/dsv41/vendor/sampler.cu b/src/dsv41/vendor/sampler.cu
index 996c02002..302637ec7 100644
--- a/src/dsv41/vendor/sampler.cu
+++ b/src/dsv41/vendor/sampler.cu
@@ -21,8 +21,13 @@
#endif
#include <cuda_runtime.h>
+#if defined(STRATA_METAL)
+#include <strata/gpu/runtime.hpp>
+#endif
+#include <algorithm>
#include <cmath>
+#include <vector>
#include <cstdio>
#include <cstdlib>
@@ -514,6 +519,171 @@ __global__ void __launch_bounds__(1024) sampler_kernel(const float* __restrict__
} // namespace
+
+#if defined(STRATA_METAL)
+// ---- Metal: the sampled chain on the host (unified memory: the logits are read where the GPU wrote them). The same function of
+// (row, params, t) as sampler_kernel above, in the same order and the same double arithmetic: penalties -> top_k (descending by
+// penalised logit, ties to the lower index) -> top_p -> min_p -> temperature -> one Philox draw; MODE 1 / 2 are the speculative
+// draft and verify. One row costs a partial sort of the vocabulary (~0.5 ms), against a ~70 ms decode step.
+namespace {
+inline uint32_t host_umulhi(uint32_t a, uint32_t b) { return (uint32_t) (((uint64_t) a * (uint64_t) b) >> 32); }
+inline float host_philox_uniform(uint64_t seed, uint64_t counter) {
+ uint32_t c0 = (uint32_t) counter, c1 = (uint32_t) (counter >> 32), c2 = (uint32_t) seed, c3 = (uint32_t) (seed >> 32);
+ for (int i = 0; i < 10; ++i) {
+ const uint32_t hi0 = host_umulhi(0x9E3779B9u, c0), hi1 = host_umulhi(0xBB67AE85u, c2);
+ const uint32_t lo0 = 0x9E3779B9u * c0, lo1 = 0xBB67AE85u * c2;
+ const uint32_t n0 = hi1 ^ c1 ^ (uint32_t) i, n1 = lo1, n2 = hi0 ^ c3 ^ 0u, n3 = lo0;
+ c0 = n0; c1 = n1; c2 = n2; c3 = n3;
+ }
+ return (float) (c0 >> 8) * (1.0f / 16777216.0f);
+}
+inline float host_penalised(float logit, int count, const SamplerParams& p) {
+ if (count <= 0) return logit;
+ if (logit <= 0.0f) logit *= p.penalty_repeat;
+ else logit /= p.penalty_repeat;
+ logit -= (float) count * p.penalty_freq + (count > 0 ? 1.0f : 0.0f) * p.penalty_present;
+ return logit;
+}
+
+// MODE 0 draw, 1 draft (kept list into ql), 2 verify (draft[t] against ql[t])
+template <int MODE>
+void host_sample(const float* dlogits, int n_tokens, int n_vocab, const int* dhistory, int history_len, const SamplerParams& p,
+ int* dout, SpecList* dql, const int* ddraft) {
+ cudaDeviceSynchronize(); // (the GPU's logits are complete, in the shared memory)
+ const float* logits = (const float*) strata::gpu::host_view((void*) dlogits);
+ const int* history = dhistory ? (const int*) strata::gpu::host_view((void*) dhistory) : nullptr;
+ int* out = (int*) strata::gpu::host_view(dout);
+ SpecList* ql = dql ? (SpecList*) strata::gpu::host_view(dql) : nullptr;
+ const int* draft = ddraft ? (const int*) strata::gpu::host_view((void*) ddraft) : nullptr;
+ const float* bias = p.logit_bias ? (const float*) strata::gpu::host_view((void*) p.logit_bias) : nullptr;
+ if (logits == nullptr || out == nullptr) {
+ std::fprintf(stderr, "sample (host): no host view of the logits / output\n");
+ std::exit(1);
+ }
+ const float inv_t = p.temperature > 0.0f ? 1.0f / p.temperature : 0.0f;
+ const int KMAX = 64;
+ int k = (p.top_k > 0 && p.top_k < KMAX) ? p.top_k : KMAX;
+ if (k > n_vocab) k = n_vocab;
+ std::vector<float> sc((size_t) n_vocab);
+ std::vector<int> idx((size_t) n_vocab);
+ for (int t = 0; t < n_tokens; ++t) {
+ const float* l = logits + (size_t) t * n_vocab;
+ const int* hrow = history ? history + (size_t) t * history_len : nullptr;
+ int hlen = 0;
+ if (hrow) {
+ hlen = p.penalty_last_n < history_len ? p.penalty_last_n : history_len;
+ if (hlen < 0) hlen = 0;
+ hrow += history_len - hlen;
+ }
+ for (int v = 0; v < n_vocab; ++v) {
+ int cnt = 0;
+ if (hlen > 0)
+ for (int i = 0; i < hlen; ++i) cnt += hrow[i] == v;
+ sc[(size_t) v] = host_penalised(bias ? l[v] + bias[v] : l[v], cnt, p);
+ }
+ // the ordered key: NaN below everything, -0 as +0 (the kernel's ordered())
+ auto less_than = [&](int a, int b) { // a ranks before b
+ const float x = sc[(size_t) a], y = sc[(size_t) b];
+ const bool nx = x != x, ny = y != y;
+ if (nx != ny) return ny;
+ if (nx && ny) return a < b;
+ if (x != y) return x > y;
+ return a < b;
+ };
+ for (int v = 0; v < n_vocab; ++v) idx[(size_t) v] = v;
+ std::nth_element(idx.begin(), idx.begin() + (k - 1), idx.end(), less_than);
+ std::sort(idx.begin(), idx.begin() + k, less_than);
+ int sel_ids[64];
+ float sel_logit[64];
+ for (int i = 0; i < k; ++i) { sel_ids[i] = idx[(size_t) i]; sel_logit[i] = sc[(size_t) idx[(size_t) i]]; }
+ int n_keep = k;
+ float mx = sel_logit[0];
+ for (int i = 1; i < k; ++i) mx = std::max(mx, sel_logit[i]);
+ if (p.top_p < 1.0f) {
+ double sum = 0.0;
+ for (int i = 0; i < k; ++i) sum += std::exp((double) sel_logit[i] - (double) mx);
+ double cum = 0.0;
+ int cut = k;
+ for (int i = 0; i < k; ++i) {
+ cum += std::exp((double) sel_logit[i] - (double) mx) / sum;
+ if (cum >= (double) p.top_p) { cut = i + 1; break; }
+ }
+ if (cut < p.min_keep) cut = p.min_keep < k ? p.min_keep : k;
+ n_keep = cut;
+ }
+ if (p.min_p > 0.0f) {
+ const float thresh = sel_logit[0] + std::log(p.min_p);
+ for (int i = 0; i < n_keep; ++i)
+ if (sel_logit[i] < thresh) { n_keep = i; break; }
+ }
+ auto scaled = [&](int i) { return sel_logit[i] * inv_t; };
+ float smx = scaled(0);
+ for (int i = 1; i < n_keep; ++i) smx = std::max(smx, scaled(i));
+ double sum = 0.0;
+ for (int i = 0; i < n_keep; ++i) sum += std::exp((double) scaled(i) - (double) smx);
+ double w[64];
+ for (int i = 0; i < n_keep; ++i) w[i] = std::exp((double) scaled(i) - (double) smx) / sum;
+ auto draw = [&](float u) -> int {
+ double cum = 0.0;
+ int pick = sel_ids[n_keep - 1];
+ for (int i = 0; i < n_keep; ++i) {
+ cum += w[i];
+ if ((double) u < cum) { pick = sel_ids[i]; break; }
+ }
+ return pick;
+ };
+ if (MODE == 0) {
+ out[t] = draw(host_philox_uniform(p.seed, p.counter + (uint64_t) t));
+ } else if (MODE == 1) {
+ SpecList* q = ql + t;
+ q->n = n_keep;
+ for (int i = 0; i < n_keep; ++i) { q->ids[i] = sel_ids[i]; q->p[i] = w[i]; }
+ out[t] = draw(host_philox_uniform(p.seed ^ 0x5bd1e9955bd1e995ull, p.counter + (uint64_t) t));
+ } else {
+ const SpecList* q = ql + t;
+ const int d = draft[t];
+ const int qn = q->n < KMAX ? q->n : KMAX;
+ auto qof = [&](int id) -> double {
+ for (int j = 0; j < qn; ++j)
+ if (q->ids[j] == id) return q->p[j];
+ return 0.0;
+ };
+ double pd = 0.0;
+ for (int i = 0; i < n_keep; ++i)
+ if (sel_ids[i] == d) { pd = w[i]; break; }
+ const double qd = qof(d);
+ int pick = -1;
+ if (qd <= 0.0) {
+ pick = draw(host_philox_uniform(p.seed, p.counter + (uint64_t) t));
+ } else if ((double) host_philox_uniform(p.seed ^ 0x9e3779b97f4a7c15ull, p.counter + (uint64_t) t) * qd < pd) {
+ pick = d;
+ } else {
+ double r[64], z = 0.0;
+ for (int i = 0; i < n_keep; ++i) {
+ const double x = w[i] - qof(sel_ids[i]);
+ r[i] = x > 0.0 ? x : 0.0;
+ z += r[i];
+ }
+ const float u = host_philox_uniform(p.seed ^ 0xc2b2ae3d27d4eb4full, p.counter + (uint64_t) t);
+ if (z > 0.0) {
+ double cum = 0.0;
+ for (int i = 0; i < n_keep; ++i) {
+ if (r[i] <= 0.0) continue;
+ pick = sel_ids[i];
+ cum += r[i] / z;
+ if ((double) u < cum) break;
+ }
+ } else {
+ pick = draw(u);
+ }
+ }
+ out[t] = pick;
+ }
+ }
+}
+} // namespace
+#endif
+
void sample_tokens(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len,
const SamplerParams& p, int* out, void* stream) {
if (n_tokens <= 0 || n_vocab <= 0) return;
@@ -540,6 +710,10 @@ void sample_tokens(const float* logits, int n_tokens, int n_vocab, const int* hi
} else {
// The same block-per-token shape: the selection's k argmax rounds reduce inside the block. See
// `sampler_kernel`'s header for what the old one-thread-per-token launch cost.
+#if defined(STRATA_METAL)
+ host_sample<0>(logits, n_tokens, n_vocab, history, history_len, p, out, nullptr, nullptr);
+ return;
+#endif
strata_ds4_launch(sampler_kernel<0>, (unsigned) n_tokens, 1024, shmem, (cudaStream_t) stream,
logits, n_vocab, n_tokens, history, history_len, p, out, nullptr, nullptr);
}
@@ -574,6 +748,10 @@ void sample_draft(const float* logits, int n_tokens, int n_vocab, const SamplerP
void* stream) {
if (n_tokens <= 0) return;
spec_check("sample_draft", n_tokens, n_vocab, p);
+#if defined(STRATA_METAL)
+ host_sample<1>(logits, n_tokens, n_vocab, nullptr, 0, p, out, q, nullptr);
+ return;
+#endif
strata_ds4_launch(sampler_kernel<1>, (unsigned) n_tokens, 1024, 0, (cudaStream_t) stream, logits, n_vocab, n_tokens, nullptr, 0,
p, out, q, nullptr);
spec_launched("sample_draft", stream);
@@ -583,6 +761,10 @@ void sample_verify(const float* logits, int n_tokens, int n_vocab, const Sampler
const int* draft, int* out, void* stream) {
if (n_tokens <= 0) return;
spec_check("sample_verify", n_tokens, n_vocab, p);
+#if defined(STRATA_METAL)
+ host_sample<2>(logits, n_tokens, n_vocab, nullptr, 0, p, out, const_cast<SpecList*>(q), draft);
+ return;
+#endif
strata_ds4_launch(sampler_kernel<2>, (unsigned) n_tokens, 1024, 0, (cudaStream_t) stream, logits, n_vocab, n_tokens, nullptr, 0,
p, out, const_cast<SpecList*>(q), draft);
spec_launched("sample_verify", stream);