Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek Metal: drafter bring-up (ring restore, host sampler for temperature and speculative sampling), per-dispatch GPU timeline
Niko1221f29ff5a59a10
diff --git a/metal/src/mtl_rt.mm b/metal/src/mtl_rt.mmindex 93c75be27..12816ab0d 100644--- a/metal/src/mtl_rt.mm+++ b/metal/src/mtl_rt.mm@@ -13,6 +13,7 @@#include <unordered_map>#include <string>#include <deque>+#include <mach/mach_time.h>#include <map>#include <thread>#include <cstdio>@@ -66,8 +67,150 @@ static inline id<MTLBuffer> BO(const Buffer& b, size_t off, size_t& o) {static inline id<MTLComputePipelineState> P(const Pipeline* p) { return (__bridge id<MTLComputePipelineState>)p->handle; }static std::string g_device_name = "unknown GPU";++// ---- DS4_TIMELINE=1: every deferred dispatch in its own encoder with start / end GPU timestamps; the report at exit gives, per kernel,+// its GPU time and the gap the GPU sat idle before it (host waits, event waits, launch). DS4_TL_FROM=N: only from dispatch N on.+static std::atomic<uint64_t> g_disp_count{0}; // dispatches encoded+namespace {+struct Timeline {+ bool on = false;+ id<MTLCounterSet> tset = nil;+ std::vector<id<MTLCounterSampleBuffer>> bufs; // a sample buffer holds only ~1500 encoders: they rotate+ size_t per = 0, cap = 0, n = 0; // encoders per buffer / in all (the buffers' limit) / recorded+ long after = 0; // DS4_TL_AFTER: record from the Nth dispatch on+ std::vector<std::string> names; // per encoder+ uint64_t cpu0 = 0, gpu0 = 0;+ bool reg = false;+ bool att = false; // the encoder just made carries a sample slot+} g_tl;++id<MTLComputeCommandEncoder> tl_make_encoder(id<MTLDevice> dev, id<MTLCommandBuffer> cb, MTLDispatchType dt) {+ g_tl.att = false;+ if (!g_tl.on) return [cb computeCommandEncoderWithDispatchType:dt];+ if (g_tl.tset == nil) {+ for (id<MTLCounterSet> cs in dev.counterSets)+ if ([cs.name isEqualToString:MTLCommonCounterSetTimestamp]) g_tl.tset = cs;+ if (g_tl.tset == nil) {+ std::fprintf(stderr, "mtl: DS4_TIMELINE: no timestamp counter set\n");+ g_tl.on = false;+ return [cb computeCommandEncoderWithDispatchType:dt];+ }+ for (size_t c = 4096; c >= 64 && g_tl.per == 0; c /= 2) {+ MTLCounterSampleBufferDescriptor* d = [MTLCounterSampleBufferDescriptor new];+ d.counterSet = g_tl.tset;+ d.storageMode = MTLStorageModeShared;+ d.sampleCount = c * 2;+ NSError* err = nil;+ id<MTLCounterSampleBuffer> b = [dev newCounterSampleBufferWithDescriptor:d error:&err];+ if (b != nil) {+ g_tl.per = c;+ g_tl.bufs.push_back(b);+ }+ }+ if (g_tl.per == 0) {+ std::fprintf(stderr, "mtl: DS4_TIMELINE: no counter sample buffer\n");+ g_tl.on = false;+ return [cb computeCommandEncoderWithDispatchType:dt];+ }+ g_tl.cap = g_tl.per * 120;+ MTLTimestamp c0 = 0, g0 = 0;+ [dev sampleTimestamps:&c0 gpuTimestamp:&g0];+ g_tl.cpu0 = c0;+ g_tl.gpu0 = g0;+ g_tl.names.assign(g_tl.cap, std::string());+ g_tl.after = std::getenv("DS4_TL_AFTER") ? std::atol(std::getenv("DS4_TL_AFTER")) : 0;+ }+ if (g_tl.n >= g_tl.cap || (long) g_disp_count.load() < g_tl.after) return [cb computeCommandEncoderWithDispatchType:dt];+ const size_t bi = g_tl.n / g_tl.per, si = g_tl.n % g_tl.per;+ if (bi >= g_tl.bufs.size()) {+ MTLCounterSampleBufferDescriptor* d = [MTLCounterSampleBufferDescriptor new];+ d.counterSet = g_tl.tset;+ d.storageMode = MTLStorageModeShared;+ d.sampleCount = g_tl.per * 2;+ NSError* err = nil;+ id<MTLCounterSampleBuffer> b = [dev newCounterSampleBufferWithDescriptor:d error:&err];+ if (b == nil) {+ g_tl.cap = g_tl.n;+ return [cb computeCommandEncoderWithDispatchType:dt];+ }+ g_tl.bufs.push_back(b);+ }+ MTLComputePassDescriptor* pd = [MTLComputePassDescriptor computePassDescriptor];+ pd.dispatchType = dt;+ pd.sampleBufferAttachments[0].sampleBuffer = g_tl.bufs[bi];+ pd.sampleBufferAttachments[0].startOfEncoderSampleIndex = 2 * si;+ pd.sampleBufferAttachments[0].endOfEncoderSampleIndex = 2 * si + 1;+ g_tl.att = true;+ return [cb computeCommandEncoderWithDescriptor:pd];+}++void tl_report(id<MTLDevice> dev) {+ if (!g_tl.on && g_tl.bufs.empty()) return;+ const size_t n = std::min(g_tl.n, g_tl.cap);+ if (n < 2) return;+ std::vector<MTLCounterResultTimestamp> tsv(2 * n);+ for (size_t b = 0; b * g_tl.per < n; ++b) {+ const size_t cnt = std::min(g_tl.per, n - b * g_tl.per);+ NSData* data = [g_tl.bufs[b] resolveCounterRange:NSMakeRange(0, 2 * cnt)];+ if (data == nil) { std::fprintf(stderr, "mtl: DS4_TIMELINE: resolve failed\n"); return; }+ std::memcpy(&tsv[2 * b * g_tl.per], [data bytes], 2 * cnt * sizeof(MTLCounterResultTimestamp));+ }+ const MTLCounterResultTimestamp* ts = tsv.data();+ MTLTimestamp c1 = 0, g1 = 0;+ [dev sampleTimestamps:&c1 gpuTimestamp:&g1];+ mach_timebase_info_data_t tb;+ mach_timebase_info(&tb);+ const double cpu_ns_per_tick = (double) tb.numer / (double) tb.denom;+ (void) cpu_ns_per_tick; // (sampleTimestamps' CPU value is in ns already)+ const double ns_per_gpu = (g1 > g_tl.gpu0) ? (double) (c1 - g_tl.cpu0) / (double) (g1 - g_tl.gpu0) : 1.0;+ std::fprintf(stderr, "timeline: cap %zu n %zu scale %.6f ns/tick first ts %llu %llu last %llu\n", g_tl.cap, g_tl.n, ns_per_gpu, (unsigned long long) ts[0].timestamp, (unsigned long long) ts[1].timestamp, (unsigned long long) ts[2 * n - 1].timestamp);+ size_t from = std::getenv("DS4_TL_FROM") ? (size_t) std::atoll(std::getenv("DS4_TL_FROM")) : 0;+ struct Acc { double busy = 0, gap = 0, gap_big = 0; size_t n = 0, n_big = 0; };+ std::map<std::string, Acc> acc;+ double t_first = 0, t_last = 0, total_busy = 0, total_gap = 0, total_big = 0;+ double prev_end = -1;+ size_t used = 0;+ for (size_t k = from; k < n; ++k) {+ if (g_tl.names[k].empty()) continue;+ const double st = (double) ts[2 * k].timestamp * ns_per_gpu * 1e-3, en = (double) ts[2 * k + 1].timestamp * ns_per_gpu * 1e-3; // us+ if (!(en >= st)) continue;+ if (prev_end < 0) t_first = st;+ Acc& a = acc[g_tl.names[k]];+ a.busy += en - st;+ ++a.n;+ total_busy += en - st;+ if (prev_end >= 0 && st >= prev_end) {+ const double gp = st - prev_end;+ a.gap += gp;+ total_gap += gp;+ if (gp > 300.0) { a.gap_big += gp; ++a.n_big; total_big += gp; }+ }+ prev_end = en;+ t_last = en;+ ++used;+ }+ std::vector<std::pair<double, std::string>> v;+ for (auto& kv : acc) v.emplace_back(kv.second.busy + kv.second.gap, kv.first);+ std::sort(v.begin(), v.end(), [](auto& x, auto& y) { return x.first > y.first; });+ std::fprintf(stderr, "timeline: %zu dispatches, wall %.1f ms, GPU busy %.1f ms, idle gaps %.1f ms (of it > 300 us: %.1f ms)\n", used,+ (t_last - t_first) / 1000.0, total_busy / 1000.0, total_gap / 1000.0, total_big / 1000.0);+ std::fprintf(stderr, "timeline: kernel n busy ms us/disp gap ms gap/disp big-gap ms (n)\n");+ for (size_t i = 0; i < v.size() && i < 40; ++i) {+ const Acc& a = acc[v[i].second];+ std::fprintf(stderr, "timeline: %-28s %6zu %9.1f %8.1f %8.1f %8.1f %10.1f (%zu)\n", v[i].second.c_str(), a.n, a.busy / 1000.0, a.busy / (double) a.n,+ a.gap / 1000.0, a.gap / (double) a.n, a.gap_big / 1000.0, a.n_big);+ }+}+} // namespace+Context::Context() : impl_(new Impl) {impl_->dev = MTLCreateSystemDefaultDevice();+ if (std::getenv("DS4_TIMELINE") && !g_tl.reg) {+ g_tl.on = true;+ g_tl.reg = true;+ static id<MTLDevice> tl_dev = impl_->dev;+ std::atexit([] { tl_report(tl_dev); });+ }if (!impl_->dev) {std::fprintf(stderr, "mtl: no Metal device\n");std::exit(2);@@ -420,7 +563,6 @@ static std::atomic<uint64_t> g_wait_ns{0}, g_enc_ns{0}, g_wait_n{0}; // host tstatic std::mutex g_prof_mu;static std::unordered_map<std::string, std::pair<double, uint64_t>> g_prof; // DS4_PROF: kernel -> (GPU us, dispatches)static std::string g_prof_last;-static std::atomic<uint64_t> g_disp_count{0}; // dispatches encoded, and command buffers committedstatic std::atomic<uint64_t> g_cb_count{0};static bool g_count_reg = false;static void ds4_report_counts() {@@ -551,10 +693,11 @@ double Context::run(const std::vector<Dispatch>& ds, int reps, bool concurrent,} elseimpl_->cb_def = [impl_->q commandBufferWithUnretainedReferences];if (impl_->qev != nil && impl_->qev_v > 0) [impl_->cb_def encodeWaitForEvent:impl_->qev value:impl_->qev_v];- impl_->enc_def = [impl_->cb_def computeCommandEncoderWithDispatchType:std::getenv("DS4_CONCURRENT") ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial];+ impl_->enc_def = tl_make_encoder(impl_->dev, impl_->cb_def, std::getenv("DS4_CONCURRENT") ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial);}const auto te0 = std::chrono::steady_clock::now();encode(impl_->enc_def, ds[0]);+ if (g_tl.on && g_tl.att && g_tl.n < g_tl.cap) { g_tl.names[g_tl.n++] = ds[0].p->name; g_tl.att = false; }g_last_act_ns.store(ka_now_ns(), std::memory_order_relaxed);g_recent.push_back(ds[0].p->name); if (g_recent.size() > 8) g_recent.pop_front();if (std::getenv("DS4_ERRINFO")) impl_->def_names += ds[0].p->name + " ";@@ -566,10 +709,10 @@ double Context::run(const std::vector<Dispatch>& ds, int reps, bool concurrent,// second matvec's activation quantiser read the previous layer's x).static const bool no_mb = std::getenv("DS4_NO_MBARRIER") != nullptr;if (!no_mb) [impl_->enc_def memoryBarrierWithScope:MTLBarrierScopeBuffers];- static const bool enc_each = std::getenv("DS4_ENC_EACH") != nullptr; // a new encoder per dispatch (debugging)+ static const bool enc_each = std::getenv("DS4_ENC_EACH") != nullptr || g_tl.on; // a new encoder per dispatch (debugging, the timeline)if (enc_each) {[impl_->enc_def endEncoding];- impl_->enc_def = [impl_->cb_def computeCommandEncoderWithDispatchType:(std::getenv("DS4_CONCURRENT") ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial)];+ impl_->enc_def = tl_make_encoder(impl_->dev, impl_->cb_def, (std::getenv("DS4_CONCURRENT") ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial));}++impl_->def_n;++g_disp_count;diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cuindex f776f88fc..f7edf5745 100644--- a/src/dsv41/ds4_kernels.cu+++ b/src/dsv41/ds4_kernels.cu@@ -2196,10 +2196,26 @@ void ext_to_ring(const uint16_t* ext, uint8_t* ring, int64_t p0, int n, int T, istrata_ds4_launch(ext_to_ring_kernel, T, kThreads, 0, s, ext, ring, (long long) p0, n, T, d, backup);}void ring_restore(uint8_t* ring, const uint8_t* backup, int64_t p0, int n, int keep, int T, int d, cudaStream_t s) {+#if defined(STRATA_METAL)+ { // (row copies on the context queue: the drafter's rollback is a few rows)+ const size_t rb = kv_row_bytes(kKvFp8, d);+ for (int t = keep; t < T; ++t) cudaMemcpyAsync(ring + (size_t) ((p0 + t) % n) * rb, backup + (size_t) t * rb, rb, cudaMemcpyDeviceToDevice, s);+ return;+ }+#endifif (T > keep)strata_ds4_launch(ring_restore_kernel, T - keep, kThreads, 0, s, ring, backup, (long long) p0, n, keep, kv_row_bytes(kKvFp8, d));}void ring_restore_batch(const RingBatch& b, int64_t p0, int n, int keep, int T, int d, cudaStream_t s) {+#if defined(STRATA_METAL)+ {+ const size_t rb = kv_row_bytes(kKvFp8, d);+ for (int li = 0; li < b.n; ++li)+ for (int t = keep; t < T; ++t)+ cudaMemcpyAsync(b.ring[li] + (size_t) ((p0 + t) % n) * rb, b.bak[li] + (size_t) t * rb, rb, cudaMemcpyDeviceToDevice, s);+ return;+ }+#endifif (T > keep && b.n > 0)strata_ds4_launch(ring_restore_batch_kernel, dim3(T - keep, b.n), kThreads, 0, s, b, (long long) p0, n, keep,kv_row_bytes(kKvFp8, d));diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex b458494f6..98d008f8b 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -8391,6 +8391,7 @@ struct Model::Impl {#endifstatic const bool skip_cpu = std::getenv("DS4_SKIP_CPU") != nullptr; // (timing only: the CPU lane's experts left out)if (!skip_cpu) cpu_experts(l, h_x, ents);+ else if (const char* fk = std::getenv("DS4_FAKE_CPU_US")) std::this_thread::sleep_for(std::chrono::microseconds(std::atoi(fk))); // (timing only)std::fill(h_part, h_part + (size_t) T * D, 0.0f);for (int sl = 0; sl < nslots; ++sl) {float* o = h_part + (size_t) cpu_rows[(size_t) sl].first * D;diff --git a/src/dsv41/vendor/sampler.cu b/src/dsv41/vendor/sampler.cuindex 996c02002..302637ec7 100644--- a/src/dsv41/vendor/sampler.cu+++ b/src/dsv41/vendor/sampler.cu@@ -21,8 +21,13 @@#endif#include <cuda_runtime.h>+#if defined(STRATA_METAL)+#include <strata/gpu/runtime.hpp>+#endif+#include <algorithm>#include <cmath>+#include <vector>#include <cstdio>#include <cstdlib>@@ -514,6 +519,171 @@ __global__ void __launch_bounds__(1024) sampler_kernel(const float* __restrict__} // namespace++#if defined(STRATA_METAL)+// ---- Metal: the sampled chain on the host (unified memory: the logits are read where the GPU wrote them). The same function of+// (row, params, t) as sampler_kernel above, in the same order and the same double arithmetic: penalties -> top_k (descending by+// penalised logit, ties to the lower index) -> top_p -> min_p -> temperature -> one Philox draw; MODE 1 / 2 are the speculative+// draft and verify. One row costs a partial sort of the vocabulary (~0.5 ms), against a ~70 ms decode step.+namespace {+inline uint32_t host_umulhi(uint32_t a, uint32_t b) { return (uint32_t) (((uint64_t) a * (uint64_t) b) >> 32); }+inline float host_philox_uniform(uint64_t seed, uint64_t counter) {+ uint32_t c0 = (uint32_t) counter, c1 = (uint32_t) (counter >> 32), c2 = (uint32_t) seed, c3 = (uint32_t) (seed >> 32);+ for (int i = 0; i < 10; ++i) {+ const uint32_t hi0 = host_umulhi(0x9E3779B9u, c0), hi1 = host_umulhi(0xBB67AE85u, c2);+ const uint32_t lo0 = 0x9E3779B9u * c0, lo1 = 0xBB67AE85u * c2;+ const uint32_t n0 = hi1 ^ c1 ^ (uint32_t) i, n1 = lo1, n2 = hi0 ^ c3 ^ 0u, n3 = lo0;+ c0 = n0; c1 = n1; c2 = n2; c3 = n3;+ }+ return (float) (c0 >> 8) * (1.0f / 16777216.0f);+}+inline float host_penalised(float logit, int count, const SamplerParams& p) {+ if (count <= 0) return logit;+ if (logit <= 0.0f) logit *= p.penalty_repeat;+ else logit /= p.penalty_repeat;+ logit -= (float) count * p.penalty_freq + (count > 0 ? 1.0f : 0.0f) * p.penalty_present;+ return logit;+}++// MODE 0 draw, 1 draft (kept list into ql), 2 verify (draft[t] against ql[t])+template <int MODE>+void host_sample(const float* dlogits, int n_tokens, int n_vocab, const int* dhistory, int history_len, const SamplerParams& p,+ int* dout, SpecList* dql, const int* ddraft) {+ cudaDeviceSynchronize(); // (the GPU's logits are complete, in the shared memory)+ const float* logits = (const float*) strata::gpu::host_view((void*) dlogits);+ const int* history = dhistory ? (const int*) strata::gpu::host_view((void*) dhistory) : nullptr;+ int* out = (int*) strata::gpu::host_view(dout);+ SpecList* ql = dql ? (SpecList*) strata::gpu::host_view(dql) : nullptr;+ const int* draft = ddraft ? (const int*) strata::gpu::host_view((void*) ddraft) : nullptr;+ const float* bias = p.logit_bias ? (const float*) strata::gpu::host_view((void*) p.logit_bias) : nullptr;+ if (logits == nullptr || out == nullptr) {+ std::fprintf(stderr, "sample (host): no host view of the logits / output\n");+ std::exit(1);+ }+ const float inv_t = p.temperature > 0.0f ? 1.0f / p.temperature : 0.0f;+ const int KMAX = 64;+ int k = (p.top_k > 0 && p.top_k < KMAX) ? p.top_k : KMAX;+ if (k > n_vocab) k = n_vocab;+ std::vector<float> sc((size_t) n_vocab);+ std::vector<int> idx((size_t) n_vocab);+ for (int t = 0; t < n_tokens; ++t) {+ const float* l = logits + (size_t) t * n_vocab;+ const int* hrow = history ? history + (size_t) t * history_len : nullptr;+ int hlen = 0;+ if (hrow) {+ hlen = p.penalty_last_n < history_len ? p.penalty_last_n : history_len;+ if (hlen < 0) hlen = 0;+ hrow += history_len - hlen;+ }+ for (int v = 0; v < n_vocab; ++v) {+ int cnt = 0;+ if (hlen > 0)+ for (int i = 0; i < hlen; ++i) cnt += hrow[i] == v;+ sc[(size_t) v] = host_penalised(bias ? l[v] + bias[v] : l[v], cnt, p);+ }+ // the ordered key: NaN below everything, -0 as +0 (the kernel's ordered())+ auto less_than = [&](int a, int b) { // a ranks before b+ const float x = sc[(size_t) a], y = sc[(size_t) b];+ const bool nx = x != x, ny = y != y;+ if (nx != ny) return ny;+ if (nx && ny) return a < b;+ if (x != y) return x > y;+ return a < b;+ };+ for (int v = 0; v < n_vocab; ++v) idx[(size_t) v] = v;+ std::nth_element(idx.begin(), idx.begin() + (k - 1), idx.end(), less_than);+ std::sort(idx.begin(), idx.begin() + k, less_than);+ int sel_ids[64];+ float sel_logit[64];+ for (int i = 0; i < k; ++i) { sel_ids[i] = idx[(size_t) i]; sel_logit[i] = sc[(size_t) idx[(size_t) i]]; }+ int n_keep = k;+ float mx = sel_logit[0];+ for (int i = 1; i < k; ++i) mx = std::max(mx, sel_logit[i]);+ if (p.top_p < 1.0f) {+ double sum = 0.0;+ for (int i = 0; i < k; ++i) sum += std::exp((double) sel_logit[i] - (double) mx);+ double cum = 0.0;+ int cut = k;+ for (int i = 0; i < k; ++i) {+ cum += std::exp((double) sel_logit[i] - (double) mx) / sum;+ if (cum >= (double) p.top_p) { cut = i + 1; break; }+ }+ if (cut < p.min_keep) cut = p.min_keep < k ? p.min_keep : k;+ n_keep = cut;+ }+ if (p.min_p > 0.0f) {+ const float thresh = sel_logit[0] + std::log(p.min_p);+ for (int i = 0; i < n_keep; ++i)+ if (sel_logit[i] < thresh) { n_keep = i; break; }+ }+ auto scaled = [&](int i) { return sel_logit[i] * inv_t; };+ float smx = scaled(0);+ for (int i = 1; i < n_keep; ++i) smx = std::max(smx, scaled(i));+ double sum = 0.0;+ for (int i = 0; i < n_keep; ++i) sum += std::exp((double) scaled(i) - (double) smx);+ double w[64];+ for (int i = 0; i < n_keep; ++i) w[i] = std::exp((double) scaled(i) - (double) smx) / sum;+ auto draw = [&](float u) -> int {+ double cum = 0.0;+ int pick = sel_ids[n_keep - 1];+ for (int i = 0; i < n_keep; ++i) {+ cum += w[i];+ if ((double) u < cum) { pick = sel_ids[i]; break; }+ }+ return pick;+ };+ if (MODE == 0) {+ out[t] = draw(host_philox_uniform(p.seed, p.counter + (uint64_t) t));+ } else if (MODE == 1) {+ SpecList* q = ql + t;+ q->n = n_keep;+ for (int i = 0; i < n_keep; ++i) { q->ids[i] = sel_ids[i]; q->p[i] = w[i]; }+ out[t] = draw(host_philox_uniform(p.seed ^ 0x5bd1e9955bd1e995ull, p.counter + (uint64_t) t));+ } else {+ const SpecList* q = ql + t;+ const int d = draft[t];+ const int qn = q->n < KMAX ? q->n : KMAX;+ auto qof = [&](int id) -> double {+ for (int j = 0; j < qn; ++j)+ if (q->ids[j] == id) return q->p[j];+ return 0.0;+ };+ double pd = 0.0;+ for (int i = 0; i < n_keep; ++i)+ if (sel_ids[i] == d) { pd = w[i]; break; }+ const double qd = qof(d);+ int pick = -1;+ if (qd <= 0.0) {+ pick = draw(host_philox_uniform(p.seed, p.counter + (uint64_t) t));+ } else if ((double) host_philox_uniform(p.seed ^ 0x9e3779b97f4a7c15ull, p.counter + (uint64_t) t) * qd < pd) {+ pick = d;+ } else {+ double r[64], z = 0.0;+ for (int i = 0; i < n_keep; ++i) {+ const double x = w[i] - qof(sel_ids[i]);+ r[i] = x > 0.0 ? x : 0.0;+ z += r[i];+ }+ const float u = host_philox_uniform(p.seed ^ 0xc2b2ae3d27d4eb4full, p.counter + (uint64_t) t);+ if (z > 0.0) {+ double cum = 0.0;+ for (int i = 0; i < n_keep; ++i) {+ if (r[i] <= 0.0) continue;+ pick = sel_ids[i];+ cum += r[i] / z;+ if ((double) u < cum) break;+ }+ } else {+ pick = draw(u);+ }+ }+ out[t] = pick;+ }+ }+}+} // namespace+#endif+void sample_tokens(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len,const SamplerParams& p, int* out, void* stream) {if (n_tokens <= 0 || n_vocab <= 0) return;@@ -540,6 +710,10 @@ void sample_tokens(const float* logits, int n_tokens, int n_vocab, const int* hi} else {// The same block-per-token shape: the selection's k argmax rounds reduce inside the block. See// `sampler_kernel`'s header for what the old one-thread-per-token launch cost.+#if defined(STRATA_METAL)+ host_sample<0>(logits, n_tokens, n_vocab, history, history_len, p, out, nullptr, nullptr);+ return;+#endifstrata_ds4_launch(sampler_kernel<0>, (unsigned) n_tokens, 1024, shmem, (cudaStream_t) stream,logits, n_vocab, n_tokens, history, history_len, p, out, nullptr, nullptr);}@@ -574,6 +748,10 @@ void sample_draft(const float* logits, int n_tokens, int n_vocab, const SamplerPvoid* stream) {if (n_tokens <= 0) return;spec_check("sample_draft", n_tokens, n_vocab, p);+#if defined(STRATA_METAL)+ host_sample<1>(logits, n_tokens, n_vocab, nullptr, 0, p, out, q, nullptr);+ return;+#endifstrata_ds4_launch(sampler_kernel<1>, (unsigned) n_tokens, 1024, 0, (cudaStream_t) stream, logits, n_vocab, n_tokens, nullptr, 0,p, out, q, nullptr);spec_launched("sample_draft", stream);@@ -583,6 +761,10 @@ void sample_verify(const float* logits, int n_tokens, int n_vocab, const Samplerconst int* draft, int* out, void* stream) {if (n_tokens <= 0) return;spec_check("sample_verify", n_tokens, n_vocab, p);+#if defined(STRATA_METAL)+ host_sample<2>(logits, n_tokens, n_vocab, nullptr, 0, p, out, const_cast<SpecList*>(q), draft);+ return;+#endifstrata_ds4_launch(sampler_kernel<2>, (unsigned) n_tokens, 1024, 0, (cudaStream_t) stream, logits, n_vocab, n_tokens, nullptr, 0,p, out, const_cast<SpecList*>(q), draft);spec_launched("sample_verify", stream);