Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41 Windows: Engram rows read unbuffered from their unmapped file; DS4_EG_DIRECT=0 as before, DS4_EG_CHECK compares against a buffered read

Niko1221a6cbc3621ead
diff --git a/src/dsv41/ds4_engram_io.hpp b/src/dsv41/ds4_engram_io.hpp
index 2402ec1af..b68919d2c 100644
--- a/src/dsv41/ds4_engram_io.hpp
+++ b/src/dsv41/ds4_engram_io.hpp
@@ -17,6 +17,7 @@
#include <chrono>
#include <condition_variable>
#include <cstdint>
+#include <cstdio>
#include <cstring>
#include <deque>
#include <memory>
@@ -129,8 +130,10 @@ struct EgBatch {
class EgReader {
public:
~EgReader() { stop(); }
- void start(const std::vector<std::string>& paths, int threads) {
+ /// direct[i]: file i read unbuffered (Windows, a file nothing maps: os::open_unbuffered), else buffered
+ void start(const std::vector<std::string>& paths, int threads, std::vector<char> direct = {}) {
paths_ = paths;
+ direct_ = std::move(direct);
for (int i = 0; i < threads; ++i) th_.emplace_back([this] { loop(); });
}
void stop() {
@@ -141,6 +144,9 @@ public:
cv_.notify_all();
for (auto& t : th_) t.join();
th_.clear();
+ if (check_n_.load() > 0)
+ std::fprintf(stderr, "ds4 Engram check (DS4_EG_CHECK): %lld rows read unbuffered, %lld differed from a buffered read\n",
+ (long long) check_n_.load(), (long long) check_bad_.load());
}
int threads() const { return (int) th_.size(); }
void submit(EgBatch& b) {
@@ -173,7 +179,9 @@ public:
private:
void loop() {
std::vector<os::fd_t> fds(paths_.size(), os::kBadFd); // this thread's own handles, opened on first use
- std::vector<uint8_t> buf;
+ std::vector<os::fd_t> ufds(paths_.size(), os::kBadFd); // (and its unbuffered ones)
+ std::vector<uint8_t> buf, abuf(3 * os::kDirectAlign);
+ uint8_t* al = abuf.data() + (os::kDirectAlign - (uintptr_t) abuf.data() % os::kDirectAlign) % os::kDirectAlign;
for (;;) {
Job job;
{
@@ -184,7 +192,7 @@ private:
q_.pop_front();
}
EgBatch& b = *job.b;
- for (int i = job.i0; i < job.i1; ++i) b.ok[(size_t) i] = read(fds, b.reads[(size_t) i], buf) ? 1 : 0;
+ for (int i = job.i0; i < job.i1; ++i) b.ok[(size_t) i] = read(fds, ufds, al, b.reads[(size_t) i], buf) ? 1 : 0;
const int64_t t = now_ns();
for (int64_t prev = b.t1.load(); t > prev && !b.t1.compare_exchange_weak(prev, t);) {
}
@@ -194,8 +202,36 @@ private:
}
}
for (os::fd_t f : fds) os::close_fd(f);
+ for (os::fd_t f : ufds) os::close_fd(f);
}
- bool read(std::vector<os::fd_t>& fds, const EgRead& r, std::vector<uint8_t>& buf) {
+ bool read(std::vector<os::fd_t>& fds, std::vector<os::fd_t>& ufds, uint8_t* al, const EgRead& r,
+ std::vector<uint8_t>& buf) {
+ if ((r.mx8 || r.n2 == 0) && r.file >= 0 && (size_t) r.file < direct_.size() && direct_[(size_t) r.file] &&
+ os::direct_span(r.off, r.n) <= 2 * os::kDirectAlign) {
+ os::fd_t& u = ufds[(size_t) r.file];
+ if (u == os::kBadFd) u = os::open_unbuffered(paths_[(size_t) r.file]);
+ if (u != os::kBadFd && os::pread_unbuffered(u, al, r.n, r.off)) {
+ const uint8_t* src = al + r.off % os::kDirectAlign;
+ static const bool check = std::getenv("DS4_EG_CHECK") != nullptr; // (each row read buffered too)
+ if (check) {
+ std::vector<uint8_t> ref(r.n);
+ os::fd_t& f2 = fds[(size_t) r.file];
+ if (f2 == os::kBadFd) f2 = os::open_ro(paths_[(size_t) r.file]);
+ const bool ok = f2 != os::kBadFd && os::pread_full(f2, ref.data(), r.n, r.off);
+ if (!ok || std::memcmp(ref.data(), src, r.n) != 0) ++check_bad_;
+ ++check_n_;
+ }
+ if (!r.mx8) { // (a GGUF's Q6_K / IQ4_NL row: its bytes as they are, the scales inside)
+ std::memcpy(r.dst, src, r.n);
+ return true;
+ }
+ for (uint32_t k = 0; k < r.n / 33; ++k) { // [scale, 32 codes] -> codes | scales
+ r.scales[k] = src[(size_t) k * 33];
+ std::memcpy(r.dst + (size_t) k * 32, src + (size_t) k * 33 + 1, 32);
+ }
+ return true;
+ } // (else buffered, below)
+ }
auto fd = [&](int file) {
if (file < 0 || file >= (int) fds.size()) return os::kBadFd;
if (fds[(size_t) file] == os::kBadFd) fds[(size_t) file] = os::open_ro(paths_[(size_t) file]);
@@ -219,6 +255,10 @@ private:
}
std::vector<std::string> paths_;
+ std::vector<char> direct_;
+
+public:
+ std::atomic<int64_t> check_n_{0}, check_bad_{0}; // (DS4_EG_CHECK: rows compared, rows that differed)
std::vector<std::thread> th_;
std::mutex mu_;
std::condition_variable cv_;
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index 4b307a026..29f595368 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -2403,7 +2403,30 @@ struct Model::Impl {
eg_cache.init((size_t) ((mb << 20) / row), row);
std::vector<std::string> paths;
for (const std::string& fn : pk.files) paths.push_back(pk.model_dir + "/" + fn);
- eg_io.start(paths, nth);
+ // Windows: the Engram tables' files that hold nothing else are unmapped and read unbuffered - an SSD's
+ // queue kept by the reader threads instead of the page cache's (whose reads of one file from many threads
+ // went ~255K rows a second here); unbuffered reads of a mapped file take a slow path, so only unmapped
+ // ones (DS4_EG_DIRECT=0: buffered, mapped, as before)
+ std::vector<char> eg_direct(pk.files.size(), 0);
+#if defined(_WIN32)
+ if (pk.gguf && (std::getenv("DS4_EG_DIRECT") == nullptr || std::atoi(std::getenv("DS4_EG_DIRECT")) != 0)) {
+ std::vector<char> eg_file(pk.files.size(), 0), other(pk.files.size(), 0);
+ for (const auto& kt : pk.t)
+ if (kt.second.file >= 0 && (size_t) kt.second.file < pk.files.size())
+ (kt.first.find(".engram.embed.") != std::string::npos ? eg_file : other)[(size_t) kt.second.file] = 1;
+ for (size_t i = 0; i < pk.files.size(); ++i)
+ if (eg_file[i] && !other[i]) {
+ if (i < pk.maps.size() && pk.maps[i].first != nullptr) {
+ os::unmap(pk.maps[i].first, pk.maps[i].second);
+ pk.maps[i] = {nullptr, 0};
+ }
+ eg_direct[i] = 1;
+ if (o.verbose) std::fprintf(stderr, "ds4: %s unmapped, its Engram rows read unbuffered\n",
+ pk.files[i].c_str());
+ }
+ }
+#endif
+ eg_io.start(paths, nth, eg_direct);
for (int li = 0; li < pk.eg_layers; ++li) {
eg_win.emplace_back(new EgBatch());
const std::string id = std::to_string(g.eg_layers[(size_t) li]);
diff --git a/src/dsv41/ds4_os.hpp b/src/dsv41/ds4_os.hpp
index 1901f064b..19d0722eb 100644
--- a/src/dsv41/ds4_os.hpp
+++ b/src/dsv41/ds4_os.hpp
@@ -149,6 +149,33 @@ inline bool pread_direct(fd_t f, void* dst, uint64_t n, uint64_t off) {
return got >= want;
#endif
}
+/// Windows: the file opened for unbuffered reads (FILE_FLAG_NO_BUFFERING), for a file no view of this process maps -
+/// unbuffered reads of a mapped file take a slow path there (open_direct); kBadFd elsewhere
+inline fd_t open_unbuffered(const std::string& path) {
+#if defined(_WIN32)
+ HANDLE h = CreateFileW(wide(path).c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING,
+ FILE_ATTRIBUTE_NORMAL | FILE_FLAG_NO_BUFFERING, nullptr);
+ return h == INVALID_HANDLE_VALUE ? kBadFd : (fd_t) h;
+#else
+ (void) path;
+ return kBadFd;
+#endif
+}
+/// [off, off + n) through open_unbuffered's handle (one thread's) into buf, kDirectAlign-aligned and at least
+/// direct_span(off, n) bytes: the bytes land at buf + off % kDirectAlign
+inline bool pread_unbuffered(fd_t f, void* buf, uint64_t n, uint64_t off) {
+#if defined(_WIN32)
+ const uint64_t a0 = off & ~(kDirectAlign - 1), span = direct_span(off, n);
+ OVERLAPPED ov{};
+ ov.Offset = (DWORD) a0;
+ ov.OffsetHigh = (DWORD) (a0 >> 32);
+ DWORD got = 0;
+ return ReadFile((HANDLE) f, buf, (DWORD) span, &got, &ov) && got >= (off - a0) + n;
+#else
+ (void) f, (void) buf, (void) n, (void) off;
+ return false;
+#endif
+}
/// The same file opened for reads from many threads at once (Windows runs the reads of a synchronous handle one at a
/// time, so an overlapped one; Linux: a plain descriptor, pread is concurrent anyway)
inline fd_t open_par(const std::string& path) {