Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

DeepSeek Metal: cached compute pipelines (the prompt path created one per expert copy), F_NOCACHE for the unbuffered expert reads, dispatch-seq log

Niko12216b2a530393d2
diff --git a/metal/src/mtl_rt.h b/metal/src/mtl_rt.h
index c60a0535d..1fd137109 100644
--- a/metal/src/mtl_rt.h
+++ b/metal/src/mtl_rt.h
@@ -91,6 +91,7 @@ public:
// no-copy buffer over the pages it spans, made resident. 0: not mappable (the cap on buffers is reached, or the pages are not
// ours). The first dispatch that reads cold pages waits for them (the buffer is wired on use).
uint64_t zc_map(const void* p, size_t bytes);
+ Pipeline pipeline_uncached(int lib, const std::string& fn, const Constants& c);
size_t zc_count() const;
uint64_t allocated() const { return allocated_; } // bytes of alloc() + wrap() buffers so far
diff --git a/metal/src/mtl_rt.mm b/metal/src/mtl_rt.mm
index bd0fd8499..eb766c397 100644
--- a/metal/src/mtl_rt.mm
+++ b/metal/src/mtl_rt.mm
@@ -426,6 +426,27 @@ uint64_t Context::zc_map(const void* p, size_t bytes) {
size_t Context::zc_count() const { std::lock_guard<std::mutex> lk(g_zc_mu); return g_zc.size(); }
Pipeline Context::pipeline(int lib, const std::string& fn, const Constants& c) {
+ // (cached: the host helpers ask for the same pipeline a dispatch at a time; newComputePipelineState costs ~0.3 ms and
+ // [MTLDevice name] reads the preferences - the prompt path's expert copies (3 a streamed expert) spent seconds there)
+ static std::mutex cache_mu;
+ static std::map<std::string, Pipeline> cache;
+ std::string key = std::to_string(lib) + "|" + fn;
+ for (const auto& v : c.vals) key += "|" + std::to_string(v.index) + ":" + std::to_string(v.type) + ":" + std::to_string(v.v);
+ key += "|" + std::to_string((uintptr_t) this);
+ {
+ std::lock_guard<std::mutex> lk(cache_mu);
+ auto it = cache.find(key);
+ if (it != cache.end()) return it->second;
+ }
+ Pipeline made = pipeline_uncached(lib, fn, c);
+ {
+ std::lock_guard<std::mutex> lk(cache_mu);
+ cache[key] = made;
+ }
+ return made;
+}
+
+Pipeline Context::pipeline_uncached(int lib, const std::string& fn, const Constants& c) {
NSError* err = nil;
id<MTLFunction> f;
NSString* name = [NSString stringWithUTF8String:fn.c_str()];
@@ -479,7 +500,7 @@ Pipeline Context::pipeline(int lib, const std::string& fn, const Constants& c) {
p.name += (i ? "," : "") + std::to_string(c.vals[i].index) + "=" + std::to_string((int)c.vals[i].v);
p.name += "}";
}
- g_device_name = [[impl_->dev name] UTF8String];
+ if (g_device_name == "unknown GPU") g_device_name = [[impl_->dev name] UTF8String];
return p;
}
diff --git a/src/dsv41/ds4_main.cpp b/src/dsv41/ds4_main.cpp
index 9a997e77a..4916954c7 100644
--- a/src/dsv41/ds4_main.cpp
+++ b/src/dsv41/ds4_main.cpp
@@ -975,6 +975,9 @@ int main(int argc, char** argv) {
const double t1 = now_ms();
std::fprintf(stderr, "prompt: %zu tokens in %.0f ms (%.2f tok/s, windows of %d)\n", ids.size(), t1 - t0,
1000.0 * (double) ids.size() / (t1 - t0), window);
+#if defined(STRATA_METAL)
+ if (std::getenv("DS4_SEQ_LOG")) std::fprintf(stderr, "dispatch seq after the prompt: %llu\n", ds4_dispatch_seq());
+#endif
const strata::ds4::Stats s0 = m.stats();
if (s0.eg_rows > 0)
std::fprintf(stderr, "prompt's Engram rows: %lld, %.1f%% without a read (repeats and the row cache), %lld read\n",
diff --git a/src/dsv41/ds4_os.hpp b/src/dsv41/ds4_os.hpp
index 0038ff618..208c02fd5 100644
--- a/src/dsv41/ds4_os.hpp
+++ b/src/dsv41/ds4_os.hpp
@@ -77,7 +77,13 @@ inline fd_t open_direct(const std::string& path) {
return h == INVALID_HANDLE_VALUE ? kBadFd : (fd_t) h;
#else
const int fd = ::open(path.c_str(), O_RDONLY | O_DIRECT);
- return fd < 0 ? kBadFd : (fd_t) fd;
+ if (fd < 0) return kBadFd;
+#if defined(__APPLE__)
+ // (O_DIRECT is 0 here: without F_NOCACHE these reads went through the unified buffer cache at 3.4 GB/s; the disk does 10-18 GB/s
+ // unbuffered, and the experts' pages no longer push the CPU lane's hot ones out of the cache)
+ if (std::getenv("DS4_MAC_BUFFERED_READS") == nullptr) fcntl(fd, F_NOCACHE, 1);
+#endif
+ return (fd_t) fd;
#endif
}
/// [off, off + n) through an unbuffered descriptor into dst, a kDirectAlign-aligned buffer of at least