Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek Metal: cached compute pipelines (the prompt path created one per expert copy), F_NOCACHE for the unbuffered expert reads, dispatch-seq log
Niko12216b2a530393d2
diff --git a/metal/src/mtl_rt.h b/metal/src/mtl_rt.hindex c60a0535d..1fd137109 100644--- a/metal/src/mtl_rt.h+++ b/metal/src/mtl_rt.h@@ -91,6 +91,7 @@ public:// no-copy buffer over the pages it spans, made resident. 0: not mappable (the cap on buffers is reached, or the pages are not// ours). The first dispatch that reads cold pages waits for them (the buffer is wired on use).uint64_t zc_map(const void* p, size_t bytes);+ Pipeline pipeline_uncached(int lib, const std::string& fn, const Constants& c);size_t zc_count() const;uint64_t allocated() const { return allocated_; } // bytes of alloc() + wrap() buffers so fardiff --git a/metal/src/mtl_rt.mm b/metal/src/mtl_rt.mmindex bd0fd8499..eb766c397 100644--- a/metal/src/mtl_rt.mm+++ b/metal/src/mtl_rt.mm@@ -426,6 +426,27 @@ uint64_t Context::zc_map(const void* p, size_t bytes) {size_t Context::zc_count() const { std::lock_guard<std::mutex> lk(g_zc_mu); return g_zc.size(); }Pipeline Context::pipeline(int lib, const std::string& fn, const Constants& c) {+ // (cached: the host helpers ask for the same pipeline a dispatch at a time; newComputePipelineState costs ~0.3 ms and+ // [MTLDevice name] reads the preferences - the prompt path's expert copies (3 a streamed expert) spent seconds there)+ static std::mutex cache_mu;+ static std::map<std::string, Pipeline> cache;+ std::string key = std::to_string(lib) + "|" + fn;+ for (const auto& v : c.vals) key += "|" + std::to_string(v.index) + ":" + std::to_string(v.type) + ":" + std::to_string(v.v);+ key += "|" + std::to_string((uintptr_t) this);+ {+ std::lock_guard<std::mutex> lk(cache_mu);+ auto it = cache.find(key);+ if (it != cache.end()) return it->second;+ }+ Pipeline made = pipeline_uncached(lib, fn, c);+ {+ std::lock_guard<std::mutex> lk(cache_mu);+ cache[key] = made;+ }+ return made;+}++Pipeline Context::pipeline_uncached(int lib, const std::string& fn, const Constants& c) {NSError* err = nil;id<MTLFunction> f;NSString* name = [NSString stringWithUTF8String:fn.c_str()];@@ -479,7 +500,7 @@ Pipeline Context::pipeline(int lib, const std::string& fn, const Constants& c) {p.name += (i ? "," : "") + std::to_string(c.vals[i].index) + "=" + std::to_string((int)c.vals[i].v);p.name += "}";}- g_device_name = [[impl_->dev name] UTF8String];+ if (g_device_name == "unknown GPU") g_device_name = [[impl_->dev name] UTF8String];return p;}diff --git a/src/dsv41/ds4_main.cpp b/src/dsv41/ds4_main.cppindex 9a997e77a..4916954c7 100644--- a/src/dsv41/ds4_main.cpp+++ b/src/dsv41/ds4_main.cpp@@ -975,6 +975,9 @@ int main(int argc, char** argv) {const double t1 = now_ms();std::fprintf(stderr, "prompt: %zu tokens in %.0f ms (%.2f tok/s, windows of %d)\n", ids.size(), t1 - t0,1000.0 * (double) ids.size() / (t1 - t0), window);+#if defined(STRATA_METAL)+ if (std::getenv("DS4_SEQ_LOG")) std::fprintf(stderr, "dispatch seq after the prompt: %llu\n", ds4_dispatch_seq());+#endifconst strata::ds4::Stats s0 = m.stats();if (s0.eg_rows > 0)std::fprintf(stderr, "prompt's Engram rows: %lld, %.1f%% without a read (repeats and the row cache), %lld read\n",diff --git a/src/dsv41/ds4_os.hpp b/src/dsv41/ds4_os.hppindex 0038ff618..208c02fd5 100644--- a/src/dsv41/ds4_os.hpp+++ b/src/dsv41/ds4_os.hpp@@ -77,7 +77,13 @@ inline fd_t open_direct(const std::string& path) {return h == INVALID_HANDLE_VALUE ? kBadFd : (fd_t) h;#elseconst int fd = ::open(path.c_str(), O_RDONLY | O_DIRECT);- return fd < 0 ? kBadFd : (fd_t) fd;+ if (fd < 0) return kBadFd;+#if defined(__APPLE__)+ // (O_DIRECT is 0 here: without F_NOCACHE these reads went through the unified buffer cache at 3.4 GB/s; the disk does 10-18 GB/s+ // unbuffered, and the experts' pages no longer push the CPU lane's hot ones out of the cache)+ if (std::getenv("DS4_MAC_BUFFERED_READS") == nullptr) fcntl(fd, F_NOCACHE, 1);+#endif+ return (fd_t) fd;#endif}/// [off, off + n) through an unbuffered descriptor into dst, a kDirectAlign-aligned buffer of at least