Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek Metal: prompt flow reads experts straight into the ring (unified memory: no staging, no DMA), head matvec 8 columns a pass
Niko122197c091c5be7f
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 0ac0f4bbe..d35d69e99 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -559,6 +559,7 @@ struct Chunk {struct Item {int il = 0, k = 0, e = 0, v = 0, stage = -1; // layer, chunk, expert, virtual slot, staging slot (-1: RAM)int mi = 0; // the device's member index in the layer's group+ bool direct = false; // (Metal) read straight into its ring slot: no staging, no DMAconst uint8_t* src[3] = {nullptr, nullptr, nullptr}; // where its gate / up / down (its parts) lie once readconst uint8_t* ram = nullptr; // its RAM-tier blob, copied into staging (not pinned there)uint8_t* keep = nullptr; // (layer-major) its layer-cache slot: what is read kept there@@ -567,6 +568,7 @@ struct Chunk {std::atomic<bool> bad{false};};bool flow = false;+ bool direct = false; // (Metal, unified memory) the flow reads the files straight into the ringint dev = -1; // the device (index into devs) this chunk state belongs tostd::unique_ptr<IoQueue> io;std::deque<Item> items; // items[0] is item `base`; the indices below are absolute@@ -2590,6 +2592,9 @@ struct Model::Impl {if (!fl) continue;cudaSetDevice(devs[di].id);cp->flow = true;+#if defined(STRATA_METAL)+ cp->direct = std::getenv("DS4_FLOW_DIRECT") == nullptr || std::atoi(std::getenv("DS4_FLOW_DIRECT")) != 0;+#endifcp->dev = (int) di;cp->io = std::make_unique<IoQueue>(n_readers);cp->stage_item.assign((size_t) cp->S, -1);@@ -3109,6 +3114,13 @@ struct Model::Impl {it.after = (int64_t) c.lc_first - 1;}}+ if (c.direct && l.nm == 1 && !(in_ram && copy_mode)) { // (the ring slot is the destination: filled by flow_pump when its turn comes)+ it.direct = true;+ it.stage = -1;+ it.left.store(3, std::memory_order_relaxed);+ ++(in_ram ? c.it_ram : hit ? c.it_lc : c.it_file);+ return;+ }it.stage = (int) (c.n_file++ % (uint64_t) c.S);uint8_t* sb = c.stage + (size_t) it.stage * c.stage_stride;for (int r = 0; r < 3; ++r) {@@ -3231,7 +3243,38 @@ struct Model::Impl {}if (wait) break;}- if (it.stage >= 0) {+ if (it.direct) {+ const size_t R = (size_t) c.ring_n, r = c.n_read % R;+ if (c.n_read >= R) { // the slot's last use has to be computed - and finished: the CPU writes it+ if (c.n_comp <= c.n_read - R) break;+ CK(cudaEventSynchronize(c.ring_free[r]));+ }+ const Layer& l = L[(size_t) it.il];+#if defined(STRATA_METAL)+ uint8_t* dst = (uint8_t*) strata::gpu::host_view(c.ring + r * c.ring_stride); // (device addresses are the GPU's: the CPU writes through the host view)+#else+ uint8_t* dst = c.ring + r * c.ring_stride;+#endif+ const x3::ExpertLayout& lay = l.lay;+ int nj = 0;+ for (int q = 0; q < 3; ++q) {+ IoQueue::Job& j = jobs[nj++];+ j = IoQueue::Job{};+ const os::fd_t pf = pk.pfds.empty() ? os::kBadFd : pk.pfds[(size_t) l.mx_file[q]];+ j.fd = pf != os::kBadFd ? pf : pk.fds[(size_t) l.mx_file[q]];+ j.dfd = pk.dfds.empty() ? os::kBadFd : pk.dfds[(size_t) l.mx_file[q]];+ j.off = l.mx_off[q] + (uint64_t) it.e * l.mx_stride[q];+ j.len = mx_part_bytes(l, q);+ j.dst = dst + lay.trellis[q];+ j.exact = true;+ j.left = &it.left;+ j.bad = &it.bad;+ j.mem = it.ram != nullptr ? it.ram + lay.trellis[q] : nullptr;+ j.map = pk.at(l.mx_file[q], j.off);+ j.keep = it.keep != nullptr ? it.keep + lay.trellis[q] : nullptr;+ }+ c.io->push(jobs, nj);+ } else if (it.stage >= 0) {const int64_t prev = c.stage_item[(size_t) it.stage];if (prev >= 0) {if ((size_t) prev >= c.n_dma) break;@@ -3298,6 +3341,12 @@ struct Model::Impl {const Layer& l = L[(size_t) it.il];uint8_t* dst = c.ring + r * c.ring_stride;const x3::ExpertLayout& lay = l.nm > 1 ? l.hlay : l.lay;+ if (it.direct) { // (already in the slot)+ CK(cudaEventRecord(c.ring_fill[r], c.cs));+ c.stream_bytes += (double) unit(l);+ ++c.n_dma;+ continue;+ }for (int p = 0; p < 2; ++p)CK(cudaMemcpyAsync(dst + lay.trellis[p], it.src[p], mx_read_len(l, p), cudaMemcpyHostToDevice, c.cs));if (l.nm > 1) { // (the down's part: in each of its rows the member's blocks)@@ -7639,7 +7688,11 @@ struct Model::Impl {cudaSetDevice(d.id);// (an EXL3 head: as many activations at a time as its split-K partials fit the scratch - 8 for a shared head, 4// for a whole one, each pass reading the whole share; a GGUF's: 4 at a time, as before)+#if defined(STRATA_METAL)+ const int per = d.head.fmt != 0 ? 8 : std::max(1, x3::mv_cols(d.head)); // (the matvec kernels take 8 columns in one pass over the head)+#elseconst int per = d.head.fmt != 0 ? 4 : std::max(1, x3::mv_cols(d.head));+#endiffor (int c = 0; c < rows; c += per)if (!mv(d, {job(d.head, xn + (size_t) c * D, out + (size_t) c * ld, std::min(per, rows - c), D, ld)}, err))return false;diff --git a/src/dsv41/ds4_pool.hpp b/src/dsv41/ds4_pool.hppindex 767cf9d65..b5dbbee93 100644--- a/src/dsv41/ds4_pool.hpp+++ b/src/dsv41/ds4_pool.hpp@@ -145,6 +145,7 @@ public:const uint8_t* mem = nullptr; // non-null: a copy of len bytes from memory to dst instead of a readconst void* map = nullptr; // the range in the file's mapping: all in the page cache, it is read buffereduint8_t* keep = nullptr; // non-null (a read): the len bytes read copied here as well+ bool exact = false; // the len bytes land exactly at dst (a ring slot: no staging slack around them)std::atomic<bool>* cancel = nullptr; // set before the job starts: it is dropped unread (bad is set, left counted down)};explicit IoQueue(int n) {@@ -204,6 +205,21 @@ private:if (j.mem != nullptr) {std::memcpy(j.dst, j.mem, (size_t) j.len);ok = true;+ } else if (j.exact) {+ // unbuffered through a bounce buffer (the disk reads aligned spans), then the bytes to where they belong+ thread_local std::vector<uint8_t> bounce;+ const uint64_t span = os::direct_span(j.off, j.len);+ if (bounce.size() < span + os::kDirectAlign) bounce.resize((size_t) span + 2 * os::kDirectAlign);+ uint8_t* bp = (uint8_t*) (((uintptr_t) bounce.data() + os::kDirectAlign - 1) & ~(uintptr_t) (os::kDirectAlign - 1));+ if (j.dfd != os::kBadFd && !(j.map != nullptr && os::resident(j.map, (size_t) j.len)) &&+ os::pread_direct(j.dfd, bp, j.len, j.off)) {+ std::memcpy(j.dst, bp + lead, (size_t) j.len);+ ok = true;+ direct_.fetch_add(j.len, std::memory_order_relaxed);+ } else if (os::pread_par(j.fd, j.dst, j.len, j.off)) {+ ok = true;+ buffered_.fetch_add(j.len, std::memory_order_relaxed);+ }} else if (j.dfd != os::kBadFd && !(j.map != nullptr && os::resident(j.map, (size_t) j.len)) &&os::pread_direct(j.dfd, j.dst, j.len, j.off)) { // (the page cache's ones: buffered, below)ok = true;@@ -212,7 +228,7 @@ private:ok = true;buffered_.fetch_add(j.len, std::memory_order_relaxed);}- if (ok && j.keep != nullptr && j.mem == nullptr) std::memmove(j.keep, j.dst + lead, (size_t) j.len); // (keep may overlap dst: the store reads in place and moves up)+ if (ok && j.keep != nullptr && j.mem == nullptr) std::memmove(j.keep, j.exact ? j.dst : j.dst + lead, (size_t) j.len); // (keep may overlap dst: the store reads in place and moves up)busy_us_.fetch_add((uint64_t) std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() - t0).count(),std::memory_order_relaxed);