Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41 Windows: Engram rows read unbuffered from their unmapped file; DS4_EG_DIRECT=0 as before, DS4_EG_CHECK compares against a buffered read
Niko1221a6cbc3621ead
diff --git a/src/dsv41/ds4_engram_io.hpp b/src/dsv41/ds4_engram_io.hppindex 2402ec1af..b68919d2c 100644--- a/src/dsv41/ds4_engram_io.hpp+++ b/src/dsv41/ds4_engram_io.hpp@@ -17,6 +17,7 @@#include <chrono>#include <condition_variable>#include <cstdint>+#include <cstdio>#include <cstring>#include <deque>#include <memory>@@ -129,8 +130,10 @@ struct EgBatch {class EgReader {public:~EgReader() { stop(); }- void start(const std::vector<std::string>& paths, int threads) {+ /// direct[i]: file i read unbuffered (Windows, a file nothing maps: os::open_unbuffered), else buffered+ void start(const std::vector<std::string>& paths, int threads, std::vector<char> direct = {}) {paths_ = paths;+ direct_ = std::move(direct);for (int i = 0; i < threads; ++i) th_.emplace_back([this] { loop(); });}void stop() {@@ -141,6 +144,9 @@ public:cv_.notify_all();for (auto& t : th_) t.join();th_.clear();+ if (check_n_.load() > 0)+ std::fprintf(stderr, "ds4 Engram check (DS4_EG_CHECK): %lld rows read unbuffered, %lld differed from a buffered read\n",+ (long long) check_n_.load(), (long long) check_bad_.load());}int threads() const { return (int) th_.size(); }void submit(EgBatch& b) {@@ -173,7 +179,9 @@ public:private:void loop() {std::vector<os::fd_t> fds(paths_.size(), os::kBadFd); // this thread's own handles, opened on first use- std::vector<uint8_t> buf;+ std::vector<os::fd_t> ufds(paths_.size(), os::kBadFd); // (and its unbuffered ones)+ std::vector<uint8_t> buf, abuf(3 * os::kDirectAlign);+ uint8_t* al = abuf.data() + (os::kDirectAlign - (uintptr_t) abuf.data() % os::kDirectAlign) % os::kDirectAlign;for (;;) {Job job;{@@ -184,7 +192,7 @@ private:q_.pop_front();}EgBatch& b = *job.b;- for (int i = job.i0; i < job.i1; ++i) b.ok[(size_t) i] = read(fds, b.reads[(size_t) i], buf) ? 1 : 0;+ for (int i = job.i0; i < job.i1; ++i) b.ok[(size_t) i] = read(fds, ufds, al, b.reads[(size_t) i], buf) ? 1 : 0;const int64_t t = now_ns();for (int64_t prev = b.t1.load(); t > prev && !b.t1.compare_exchange_weak(prev, t);) {}@@ -194,8 +202,36 @@ private:}}for (os::fd_t f : fds) os::close_fd(f);+ for (os::fd_t f : ufds) os::close_fd(f);}- bool read(std::vector<os::fd_t>& fds, const EgRead& r, std::vector<uint8_t>& buf) {+ bool read(std::vector<os::fd_t>& fds, std::vector<os::fd_t>& ufds, uint8_t* al, const EgRead& r,+ std::vector<uint8_t>& buf) {+ if ((r.mx8 || r.n2 == 0) && r.file >= 0 && (size_t) r.file < direct_.size() && direct_[(size_t) r.file] &&+ os::direct_span(r.off, r.n) <= 2 * os::kDirectAlign) {+ os::fd_t& u = ufds[(size_t) r.file];+ if (u == os::kBadFd) u = os::open_unbuffered(paths_[(size_t) r.file]);+ if (u != os::kBadFd && os::pread_unbuffered(u, al, r.n, r.off)) {+ const uint8_t* src = al + r.off % os::kDirectAlign;+ static const bool check = std::getenv("DS4_EG_CHECK") != nullptr; // (each row read buffered too)+ if (check) {+ std::vector<uint8_t> ref(r.n);+ os::fd_t& f2 = fds[(size_t) r.file];+ if (f2 == os::kBadFd) f2 = os::open_ro(paths_[(size_t) r.file]);+ const bool ok = f2 != os::kBadFd && os::pread_full(f2, ref.data(), r.n, r.off);+ if (!ok || std::memcmp(ref.data(), src, r.n) != 0) ++check_bad_;+ ++check_n_;+ }+ if (!r.mx8) { // (a GGUF's Q6_K / IQ4_NL row: its bytes as they are, the scales inside)+ std::memcpy(r.dst, src, r.n);+ return true;+ }+ for (uint32_t k = 0; k < r.n / 33; ++k) { // [scale, 32 codes] -> codes | scales+ r.scales[k] = src[(size_t) k * 33];+ std::memcpy(r.dst + (size_t) k * 32, src + (size_t) k * 33 + 1, 32);+ }+ return true;+ } // (else buffered, below)+ }auto fd = [&](int file) {if (file < 0 || file >= (int) fds.size()) return os::kBadFd;if (fds[(size_t) file] == os::kBadFd) fds[(size_t) file] = os::open_ro(paths_[(size_t) file]);@@ -219,6 +255,10 @@ private:}std::vector<std::string> paths_;+ std::vector<char> direct_;++public:+ std::atomic<int64_t> check_n_{0}, check_bad_{0}; // (DS4_EG_CHECK: rows compared, rows that differed)std::vector<std::thread> th_;std::mutex mu_;std::condition_variable cv_;diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 4b307a026..29f595368 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -2403,7 +2403,30 @@ struct Model::Impl {eg_cache.init((size_t) ((mb << 20) / row), row);std::vector<std::string> paths;for (const std::string& fn : pk.files) paths.push_back(pk.model_dir + "/" + fn);- eg_io.start(paths, nth);+ // Windows: the Engram tables' files that hold nothing else are unmapped and read unbuffered - an SSD's+ // queue kept by the reader threads instead of the page cache's (whose reads of one file from many threads+ // went ~255K rows a second here); unbuffered reads of a mapped file take a slow path, so only unmapped+ // ones (DS4_EG_DIRECT=0: buffered, mapped, as before)+ std::vector<char> eg_direct(pk.files.size(), 0);+#if defined(_WIN32)+ if (pk.gguf && (std::getenv("DS4_EG_DIRECT") == nullptr || std::atoi(std::getenv("DS4_EG_DIRECT")) != 0)) {+ std::vector<char> eg_file(pk.files.size(), 0), other(pk.files.size(), 0);+ for (const auto& kt : pk.t)+ if (kt.second.file >= 0 && (size_t) kt.second.file < pk.files.size())+ (kt.first.find(".engram.embed.") != std::string::npos ? eg_file : other)[(size_t) kt.second.file] = 1;+ for (size_t i = 0; i < pk.files.size(); ++i)+ if (eg_file[i] && !other[i]) {+ if (i < pk.maps.size() && pk.maps[i].first != nullptr) {+ os::unmap(pk.maps[i].first, pk.maps[i].second);+ pk.maps[i] = {nullptr, 0};+ }+ eg_direct[i] = 1;+ if (o.verbose) std::fprintf(stderr, "ds4: %s unmapped, its Engram rows read unbuffered\n",+ pk.files[i].c_str());+ }+ }+#endif+ eg_io.start(paths, nth, eg_direct);for (int li = 0; li < pk.eg_layers; ++li) {eg_win.emplace_back(new EgBatch());const std::string id = std::to_string(g.eg_layers[(size_t) li]);diff --git a/src/dsv41/ds4_os.hpp b/src/dsv41/ds4_os.hppindex 1901f064b..19d0722eb 100644--- a/src/dsv41/ds4_os.hpp+++ b/src/dsv41/ds4_os.hpp@@ -149,6 +149,33 @@ inline bool pread_direct(fd_t f, void* dst, uint64_t n, uint64_t off) {return got >= want;#endif}+/// Windows: the file opened for unbuffered reads (FILE_FLAG_NO_BUFFERING), for a file no view of this process maps -+/// unbuffered reads of a mapped file take a slow path there (open_direct); kBadFd elsewhere+inline fd_t open_unbuffered(const std::string& path) {+#if defined(_WIN32)+ HANDLE h = CreateFileW(wide(path).c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING,+ FILE_ATTRIBUTE_NORMAL | FILE_FLAG_NO_BUFFERING, nullptr);+ return h == INVALID_HANDLE_VALUE ? kBadFd : (fd_t) h;+#else+ (void) path;+ return kBadFd;+#endif+}+/// [off, off + n) through open_unbuffered's handle (one thread's) into buf, kDirectAlign-aligned and at least+/// direct_span(off, n) bytes: the bytes land at buf + off % kDirectAlign+inline bool pread_unbuffered(fd_t f, void* buf, uint64_t n, uint64_t off) {+#if defined(_WIN32)+ const uint64_t a0 = off & ~(kDirectAlign - 1), span = direct_span(off, n);+ OVERLAPPED ov{};+ ov.Offset = (DWORD) a0;+ ov.OffsetHigh = (DWORD) (a0 >> 32);+ DWORD got = 0;+ return ReadFile((HANDLE) f, buf, (DWORD) span, &got, &ov) && got >= (off - a0) + n;+#else+ (void) f, (void) buf, (void) n, (void) off;+ return false;+#endif+}/// The same file opened for reads from many threads at once (Windows runs the reads of a synchronous handle one at a/// time, so an overlapped one; Linux: a plain descriptor, pread is concurrent anyway)inline fd_t open_par(const std::string& path) {