Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
PLE reader: a failed read no longer leaves the next ticket waiting for ever; stress test for the direct table reader (#3602)
Niko1221ebe3383a3f70
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 9c6f28790..1ec4aef94 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -1505,6 +1505,14 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)target_link_libraries(ple_reader_test PRIVATE strata_kernels ${_strata_gpu_runtime_target})add_test(NAME ple_reader_selftest COMMAND ple_reader_test --selftest)+ # #3602: the same reader under load - several clients, a tiny row cache, one in-flight slot, injected delays, rows on+ # every page phase, late-writer checks, the reader after a failed read. CPU and a temp file only; 77 = no direct reads here.+ add_executable(ple_reader_stress_test src/ngram/ple_reader_stress_test.cpp)+ target_include_directories(ple_reader_stress_test PRIVATE ${_strata_gpu_include_directories})+ target_link_libraries(ple_reader_stress_test PRIVATE strata_kernels ${_strata_gpu_runtime_target})+ add_test(NAME ple_reader_stress COMMAND ple_reader_stress_test)+ set_tests_properties(ple_reader_stress PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 300)+# 0.2.7: the PLE residency rule (table + expert arena + headroom); header-only, needs nothingadd_executable(ple_residency_test src/core/ple_residency_test.cpp)target_include_directories(ple_residency_test PRIVATE include)diff --git a/src/ngram/ple_reader.cpp b/src/ngram/ple_reader.cppindex 9d2a05051..6e108ac35 100644--- a/src/ngram/ple_reader.cpp+++ b/src/ngram/ple_reader.cpp@@ -429,6 +429,13 @@ PleReader::Ticket PleReader::issue(const uint32_t* rows, size_t n, uint8_t* out_const bool rearm = m.keep_us > 0 && (m.last_issue_us <= 0 || now - m.last_issue_us > m.keep_window_us);m.last_issue_us = now;TicketState& ts = m.tickets[id];+ // After a failed read the reader stays failed (`collect` reports `error` until the next `open`) and its worker has+ // parked (worker_loop). A ticket queued now would never be read and `collect` would wait for it for ever, so it+ // is not queued: it has no jobs, and `collect` returns the error at once.+ if (!m.error.empty()) {+ ts.pending = 0;+ return Ticket{id};+ }std::unordered_map<uint64_t, size_t> by_page; // aligned offset -> index in `jobs`std::vector<Job> jobs;for (size_t i = 0; i < n; ++i) {@@ -495,7 +502,7 @@ bool PleReader::collect(Ticket t, std::string& err) {});it = m.tickets.find(t.id);if (it == m.tickets.end()) { err = "PleReader: unknown ticket"; return false; }- if (!m.error.empty()) { err = m.error; return false; }+ if (!m.error.empty()) { err = m.error; m.tickets.erase(it); return false; }m.stats.wait_us += now_us() - start;m.tickets.erase(it);return true;@@ -508,7 +515,7 @@ bool PleReader::collect(Ticket t, std::string& err) {it = m.tickets.find(t.id);if (it == m.tickets.end()) { err = "PleReader: unknown ticket"; return false; }}- if (!m.error.empty()) { err = m.error; return false; }+ if (!m.error.empty()) { err = m.error; m.tickets.erase(it); return false; }m.stats.wait_us += now_us() - start;m.tickets.erase(it);return true;diff --git a/src/ngram/ple_reader_stress_test.cpp b/src/ngram/ple_reader_stress_test.cppnew file mode 100644index 000000000..d4f977b69--- /dev/null+++ b/src/ngram/ple_reader_stress_test.cpp@@ -0,0 +1,495 @@+// src/ngram/ple_reader_stress_test.cpp - #3602: the direct table reader under load, every delivered row compared byte+// for byte with a plain buffered read of the same file. CPU and a temp file only, no model, no GPU.+//+// ple_reader_stress_test [--dir D] [--seconds S] [--big] [--no-sparse-check]+//+// What it covers that ple_reader_test --selftest does not:+// * several client threads issuing and collecting at once (the io thread, the row cache and the slot buffers are+// shared), several tickets outstanding per client, collected in any order;+// * a row cache so small that nearly every insert evicts, next to one that holds everything;+// * max_inflight 1 and 2 (a slot is reused at once), randomly injected completion delays, the keep-alive reads;+// * rows that straddle a page boundary at every phase (header offsets 0, 192, 4051, 4000 and row sizes 90, 110, 160,+// 4096), the first and last rows of the table, a file with and without bytes after the table, out-of-range rows;+// * "late writers": every collected buffer is kept and compared again after later tickets ran, so a read that+// lands in a buffer after `collect` returned shows up;+// * the reader after a failed read: the next issue/collect must fail, not wait for ever (Linux: the file is cut+// short under the open reader);+// * --big: a sparse table whose rows sit at file offsets around 2^31 and 2^32 and at the end of a 5 GiB file.+#include "strata/ngram/ple_reader.hpp"+#include "strata/platform/direct_file.hpp"++#include <algorithm>+#include <atomic>+#include <chrono>+#include <cstdint>+#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <deque>+#include <filesystem>+#include <fstream>+#include <future>+#include <mutex>+#include <random>+#include <set>+#include <string>+#include <thread>+#include <vector>++#if defined(_WIN32)+#define WIN32_LEAN_AND_MEAN+#define NOMINMAX+#include <windows.h>+#include <winioctl.h>+#else+#include <unistd.h>+#endif++namespace ng = strata::ngram;++namespace {++std::atomic<int> g_fail{0};+std::mutex g_print;+#define CHECK(c, ...) \+ do { \+ if (!(c)) { \+ std::lock_guard<std::mutex> lk_(g_print); \+ std::fprintf(stderr, "FAIL %s:%d: ", __FILE__, __LINE__); \+ std::fprintf(stderr, __VA_ARGS__); \+ std::fprintf(stderr, "\n"); \+ ++g_fail; \+ } \+ } while (0)++constexpr uint32_t PAGE = ng::PAGE;++/// The bytes of row `row` (rb bytes): a xorshift stream seeded by the row, so no two rows alike and a row read from+/// the wrong place never looks like another row's.+void fill_row(uint32_t row, uint32_t rb, uint8_t* out) {+ uint64_t x = 0x9E3779B97F4A7C15ull * (uint64_t) (row + 1) + 0x1234567ull;+ for (uint32_t b = 0; b < rb; ++b) {+ x ^= x << 13;+ x ^= x >> 7;+ x ^= x << 17;+ out[b] = (uint8_t) (x >> 24);+ }+}++struct Spec {+ const char* name;+ uint32_t rb;+ uint64_t header;+ uint32_t n_rows;+ uint64_t trailing; // bytes after the table+};++bool make_table(const std::string& path, const Spec& s) {+ std::ofstream f(path, std::ios::binary | std::ios::trunc);+ if (!f) return false;+ std::vector<uint8_t> head(s.header, 0xAB);+ if (!head.empty()) f.write((const char*) head.data(), (std::streamsize) head.size());+ std::vector<uint8_t> r(s.rb);+ for (uint32_t i = 0; i < s.n_rows; ++i) {+ fill_row(i, s.rb, r.data());+ f.write((const char*) r.data(), s.rb);+ }+ std::vector<uint8_t> tail(s.trailing, 0xEE);+ if (!tail.empty()) f.write((const char*) tail.data(), (std::streamsize) tail.size());+ return (bool) f;+}++/// A buffered read of the file: the reference.+struct Ref {+ std::ifstream f;+ uint64_t header;+ uint32_t rb;+ uint64_t n_rows;+ Ref(const std::string& path, uint64_t h, uint32_t r, uint64_t n) : f(path, std::ios::binary), header(h), rb(r), n_rows(n) {}+ void row(uint32_t r, uint8_t* out) {+ if (r >= n_rows) {+ std::memset(out, 0, rb);+ return;+ }+ f.clear();+ f.seekg((std::streamoff) (header + (uint64_t) r * rb));+ f.read((char*) out, rb);+ if ((uint32_t) f.gcount() != rb) std::memset(out, 0x77, rb); // the reference itself failed: never equal+ }+};++struct Delivered {+ std::vector<uint32_t> rows;+ std::vector<uint8_t> buf;+};++/// 0 = delivered bytes equal the reference (and, where `expect` is set, the generated row).+bool verify(Ref& ref, const Delivered& d, uint32_t rb, const char* what, bool generated) {+ std::vector<uint8_t> want(rb), gen(rb);+ for (size_t i = 0; i < d.rows.size(); ++i) {+ ref.row(d.rows[i], want.data());+ if (std::memcmp(want.data(), &d.buf[i * rb], rb) != 0) {+ size_t b = 0;+ while (b < rb && want[b] == d.buf[i * rb + b]) ++b;+ CHECK(false, "%s: row %u (index %zu of %zu) differs from the file at byte %zu", what, d.rows[i], i, d.rows.size(), b);+ return false;+ }+ if (generated && d.rows[i] < ref.n_rows) {+ fill_row(d.rows[i], rb, gen.data());+ if (std::memcmp(gen.data(), want.data(), rb) != 0) {+ CHECK(false, "%s: row %u: the file is not what the writer wrote", what, d.rows[i]);+ return false;+ }+ }+ }+ return true;+}++struct Picker {+ const Spec& s;+ std::vector<uint32_t> straddlers, tail_rows, hot;+ explicit Picker(const Spec& sp, std::mt19937& rng) : s(sp) {+ for (uint32_t r = 0; r < s.n_rows; ++r) {+ const uint64_t a = s.header + (uint64_t) r * s.rb;+ if (a / PAGE != (a + s.rb - 1) / PAGE) straddlers.push_back(r);+ }+ for (uint32_t k = 0; k < 200 && k < s.n_rows; ++k) tail_rows.push_back(s.n_rows - 1 - k);+ for (int k = 0; k < 48; ++k) hot.push_back(rng() % s.n_rows);+ }+ uint32_t one(std::mt19937& rng) const {+ switch (rng() % 9) {+ case 0: return straddlers.empty() ? rng() % s.n_rows : straddlers[rng() % straddlers.size()];+ case 1: return tail_rows[rng() % tail_rows.size()];+ case 2: return hot[rng() % hot.size()];+ case 3: return (uint32_t) (rng() % 3) + (uint32_t) (rng() % 4); // the first rows+ case 4: return s.n_rows + (uint32_t) (rng() % 4); // just out of range+ case 5: return rng() % 40 == 0 ? 0xFFFFFFFFu : rng() % s.n_rows;+ default: return rng() % s.n_rows;+ }+ }+ std::vector<uint32_t> make(std::mt19937& rng) const {+ static const size_t sizes[] = {1, 2, 5, 16, 16, 17, 64, 130, 700, 2500};+ const size_t n = sizes[rng() % (sizeof sizes / sizeof sizes[0])];+ std::vector<uint32_t> r;+ r.reserve(n);+ switch (rng() % 5) {+ case 0: { // a run: neighbours share pages+ const uint32_t at = rng() % s.n_rows;+ for (size_t i = 0; i < n; ++i) r.push_back(at + (uint32_t) i);+ break;+ }+ case 1: // duplicates+ for (size_t i = 0; i < n; ++i) r.push_back(hot[rng() % 4]);+ break;+ default:+ for (size_t i = 0; i < n; ++i) r.push_back(one(rng));+ }+ return r;+ }+};++struct Config {+ bool threaded;+ uint32_t inflight;+ uint64_t cache_rows;+ bool delay;+ bool keepalive;+ int clients;+};++void client(ng::PleReader& rd, const Spec& s, const Picker& pk, const std::string& path, uint32_t seed,+ std::chrono::steady_clock::time_point until, long min_tickets, std::atomic<long>& done) {+ std::mt19937 rng(seed);+ Ref ref(path, s.header, s.rb, s.n_rows);+ std::deque<Delivered> keep;+ long tickets = 0;+ while (std::chrono::steady_clock::now() < until || tickets < min_tickets) {+ const int k = 1 + (int) (rng() % 3);+ std::vector<Delivered> ds((size_t) k);+ std::vector<ng::PleReader::Ticket> ts((size_t) k);+ for (int j = 0; j < k; ++j) {+ ds[(size_t) j].rows = pk.make(rng);+ ds[(size_t) j].buf.assign(ds[(size_t) j].rows.size() * s.rb, 0xCC);+ ts[(size_t) j] = rd.issue(ds[(size_t) j].rows.data(), ds[(size_t) j].rows.size(), ds[(size_t) j].buf.data());+ }+ std::vector<int> order((size_t) k);+ for (int j = 0; j < k; ++j) order[(size_t) j] = j;+ std::shuffle(order.begin(), order.end(), rng);+ for (int j : order) {+ std::string err;+ if (!rd.collect(ts[(size_t) j], err)) {+ CHECK(false, "collect failed: %s", err.c_str());+ return;+ }+ verify(ref, ds[(size_t) j], s.rb, "ticket", true);+ ++tickets;+ keep.push_back(std::move(ds[(size_t) j]));+ if (keep.size() > 6) keep.pop_front();+ }+ if (rng() % 4 == 0) // late writers: what was collected earlier must still be what it was+ for (const Delivered& d : keep) verify(ref, d, s.rb, "buffer changed after collect", false);+ if (g_fail.load() > 20) return;+ }+ for (const Delivered& d : keep) verify(ref, d, s.rb, "buffer changed after collect (end)", false);+ done += tickets;+}++bool run_config(const std::string& dir, const Spec& s, const Config& c, double seconds, uint32_t seed) {+ const std::string path = dir + "/ple_stress_" + std::to_string(s.rb) + "_" + std::to_string(s.header) + ".bin";+ std::mt19937 rng(seed);+ Picker pk(s, rng);+ ng::PleReader rd;+ std::string err;+ const uint64_t cache_rows = std::min<uint64_t>(c.cache_rows, (64ull << 20) / s.rb); // at most 64 MiB of cache+ if (!rd.open(path, s.header, s.n_rows, c.inflight, cache_rows, err, c.threaded, s.rb)) {+ std::fprintf(stderr, "cannot open %s: %s\n", path.c_str(), err.c_str());+ return false;+ }+ if (c.keepalive) rd.set_keepalive(1.0, 2.0);+ std::atomic<bool> stop{false};+ std::thread chaos;+ // (the caller-thread reader has no lock: its delay is set once, from this thread, before any client starts)+ if (c.delay && !c.threaded) rd.set_injected_delay_us(300);+ if (c.delay && c.threaded)+ chaos = std::thread([&] {+ std::mt19937 r(seed ^ 77u);+ while (!stop.load()) {+ static const double d[] = {0, 0, 150, 600, 2500};+ rd.set_injected_delay_us(d[r() % 5]);+ std::this_thread::sleep_for(std::chrono::milliseconds(3));+ }+ rd.set_injected_delay_us(0);+ });+ const auto until = std::chrono::steady_clock::now() + std::chrono::milliseconds((long) (seconds * 1000));+ std::atomic<long> done{0};+ std::vector<std::thread> th;+ const int clients = c.threaded ? c.clients : 1;+ // a run that is slow (delays) still makes at least this many tickets per client+ for (int i = 0; i < clients; ++i)+ th.emplace_back(client, std::ref(rd), std::cref(s), std::cref(pk), std::cref(path), seed + 1000u * (uint32_t) i, until, 12L,+ std::ref(done));+ for (auto& t : th) t.join();+ stop = true;+ if (chaos.joinable()) chaos.join();+ CHECK(rd.cache_size() <= rd.cache_capacity(), "row cache %llu over its capacity %llu", (unsigned long long) rd.cache_size(),+ (unsigned long long) rd.cache_capacity());+ const ng::ReaderStats st = rd.snapshot();+ std::printf(" %-14s rb %3u hdr %4llu | %s inflight %2u cache %5llu%s%s clients %d: %ld tickets, %llu rows, %.1f%% cache hits, %llu reads\n",+ s.name, s.rb, (unsigned long long) s.header, c.threaded ? "thread" : "caller", c.inflight,+ (unsigned long long) c.cache_rows, c.delay ? " delay" : "", c.keepalive ? " keepalive" : "", clients,+ done.load(), (unsigned long long) st.requests, st.requests ? 100.0 * (double) st.cache_hits / (double) st.requests : 0.0,+ (unsigned long long) st.reads);+ rd.close();+ return true;+}++#if !defined(_WIN32)+/// After a failed read the reader must keep failing (collect returns false) and must not hang.+void error_does_not_hang(const std::string& dir) {+ const Spec s{"error", 90, 192, 20000, 0};+ const std::string path = dir + "/ple_stress_error.bin";+ if (!make_table(path, s)) { CHECK(false, "cannot write %s", path.c_str()); return; }+ for (bool threaded : {true, false}) {+ if (!make_table(path, s)) { CHECK(false, "cannot rewrite %s", path.c_str()); return; }+ ng::PleReader rd;+ std::string err;+ if (!rd.open(path, s.header, s.n_rows, 8, 0, err, threaded, s.rb)) { CHECK(false, "open: %s", err.c_str()); return; }+ std::vector<uint32_t> rows = {0, 1, 2, 3};+ std::vector<uint8_t> out(rows.size() * s.rb);+ CHECK(rd.collect(rd.issue(rows.data(), rows.size(), out.data()), err), "first ticket: %s", err.c_str());+ // cut the file short under the open reader: every page past the cut now reads as a short read+ if (truncate(path.c_str(), (off_t) (s.header + 100 * s.rb)) != 0) { CHECK(false, "truncate failed"); return; }+ std::vector<uint32_t> beyond = {15000, 15001, 19999};+ std::vector<uint8_t> out2(beyond.size() * s.rb);+ auto go = std::async(std::launch::async, [&]() -> int {+ std::string e1, e2, e3;+ const bool a = rd.collect(rd.issue(beyond.data(), beyond.size(), out2.data()), e1);+ // a ticket after the failure: nothing to read (or something the cache holds) - it must still return+ std::vector<uint32_t> again = {5, 15500};+ std::vector<uint8_t> out3(again.size() * s.rb);+ const bool b = rd.collect(rd.issue(again.data(), again.size(), out3.data()), e2);+ const bool c = rd.collect(rd.issue(beyond.data(), beyond.size(), out2.data()), e3);+ return (a ? 1 : 0) | (b ? 2 : 0) | (c ? 4 : 0);+ });+ if (go.wait_for(std::chrono::seconds(20)) != std::future_status::ready) {+ std::fprintf(stderr, "FAIL: a ticket issued after a failed read never returned (%s reader)\n", threaded ? "thread" : "caller");+ ++g_fail;+ std::_Exit(3); // the reader's destructor would wait for the same thing+ }+ const int r = go.get();+ CHECK((r & 1) == 0, "a read past the cut file did not fail (%s reader)", threaded ? "thread" : "caller");+ CHECK((r & 2) == 0 && (r & 4) == 0, "a ticket after the failure succeeded (mask %d, %s reader)", r, threaded ? "thread" : "caller");+ rd.close();+ }+ std::filesystem::remove(path);+ std::printf(" error path: a failed read fails the next tickets too, none hangs\n");+}+#endif++// ----------------------------------------------------------------------------------------------- the big sparse table+/// A sparse file of `size` bytes with the given (offset, bytes) pieces written.+bool make_sparse(const std::string& path, uint64_t size, const std::vector<std::pair<uint64_t, std::vector<uint8_t>>>& pieces) {+#if defined(_WIN32)+ HANDLE h = CreateFileA(path.c_str(), GENERIC_READ | GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr);+ if (h == INVALID_HANDLE_VALUE) return false;+ DWORD ret = 0;+ DeviceIoControl(h, FSCTL_SET_SPARSE, nullptr, 0, nullptr, 0, &ret, nullptr); // if it is refused the file is just big+ bool ok = true;+ for (const auto& p : pieces) {+ LARGE_INTEGER at;+ at.QuadPart = (LONGLONG) p.first;+ DWORD w = 0;+ ok = ok && SetFilePointerEx(h, at, nullptr, FILE_BEGIN) && WriteFile(h, p.second.data(), (DWORD) p.second.size(), &w, nullptr) &&+ w == p.second.size();+ }+ LARGE_INTEGER end;+ end.QuadPart = (LONGLONG) size;+ ok = ok && SetFilePointerEx(h, end, nullptr, FILE_BEGIN) && SetEndOfFile(h);+ CloseHandle(h);+ return ok;+#else+ {+ std::ofstream f(path, std::ios::binary | std::ios::trunc);+ if (!f) return false;+ for (const auto& p : pieces) {+ f.seekp((std::streamoff) p.first);+ f.write((const char*) p.second.data(), (std::streamsize) p.second.size());+ }+ if (!f) return false;+ }+ std::error_code ec;+ std::filesystem::resize_file(path, size, ec);+ return !ec;+#endif+}++void big_offsets(const std::string& dir) {+ constexpr uint32_t RB = 90;+ constexpr uint64_t HEADER = 192;+ constexpr uint32_t N = 61000000u; // 5.49e9 bytes: past 2^32 (4.29e9), below 2^33+ const uint64_t size = HEADER + (uint64_t) N * RB;+ std::set<uint32_t> marks;+ auto around = [&](uint64_t byte_offset) {+ const uint32_t r = (uint32_t) ((byte_offset - HEADER) / RB);+ for (int d = -3; d <= 3; ++d)+ if ((int64_t) r + d >= 0 && (uint64_t) ((int64_t) r + d) < N) marks.insert((uint32_t) ((int64_t) r + d));+ };+ around(1ull << 31);+ around((1ull << 32) - 4096);+ around(1ull << 32);+ around((1ull << 32) + 4096);+ around(3ull << 30);+ around(5ull << 30);+ for (uint32_t k = 0; k < 6; ++k) marks.insert(N - 1 - k);+ for (uint32_t k = 0; k < 4; ++k) marks.insert(k);+ // rows that cross a page boundary in the neighbourhood of 2^32+ for (uint32_t r = (uint32_t) (((1ull << 32) - HEADER) / RB) - 200; r < (uint32_t) (((1ull << 32) - HEADER) / RB) + 200; ++r) {+ const uint64_t a = HEADER + (uint64_t) r * RB;+ if (a / PAGE != (a + RB - 1) / PAGE) marks.insert(r);+ }+ std::vector<std::pair<uint64_t, std::vector<uint8_t>>> pieces;+ for (uint32_t r : marks) {+ std::vector<uint8_t> b(RB);+ fill_row(r, RB, b.data());+ pieces.emplace_back(HEADER + (uint64_t) r * RB, std::move(b));+ }+ const std::string path = dir + "/ple_stress_big.bin";+ if (!make_sparse(path, size, pieces)) {+ std::fprintf(stderr, " big table: cannot create the sparse file here, skipped\n");+ return;+ }+ Ref ref(path, HEADER, RB, N);+ std::mt19937 rng(99);+ for (bool threaded : {true, false}) {+ ng::PleReader rd;+ std::string err;+ if (!rd.open(path, HEADER, N, 16, 1024, err, threaded, RB)) { CHECK(false, "big open: %s", err.c_str()); break; }+ std::vector<uint32_t> mk(marks.begin(), marks.end());+ for (int round = 0; round < 40; ++round) {+ Delivered d;+ for (int i = 0; i < 64; ++i) d.rows.push_back(rng() % 3 == 0 ? (uint32_t) (rng() % N) : mk[rng() % mk.size()]);+ d.buf.assign(d.rows.size() * RB, 0xCC);+ CHECK(rd.collect(rd.issue(d.rows.data(), d.rows.size(), d.buf.data()), err), "big collect: %s", err.c_str());+ verify(ref, d, RB, "big table", false);+ // and against what was written: a marked row is its generated bytes, any other row is zeros+ std::vector<uint8_t> want(RB);+ for (size_t i = 0; i < d.rows.size(); ++i) {+ if (marks.count(d.rows[i])) fill_row(d.rows[i], RB, want.data());+ else std::memset(want.data(), 0, RB);+ CHECK(std::memcmp(want.data(), &d.buf[i * RB], RB) == 0, "big table: row %u is not what was written", d.rows[i]);+ }+ }+ rd.close();+ }+ std::error_code ec;+ std::filesystem::remove(path, ec);+ std::printf(" big table: %zu marked rows at offsets around 2^31, 2^32 and the end of a %.2f GiB file, both readers: done\n",+ marks.size(), (double) size / (1024.0 * 1024.0 * 1024.0));+}++} // namespace++int main(int argc, char** argv) {+ std::string dir = std::filesystem::temp_directory_path().string();+ double seconds = 0.3;+ bool big = false, only_error = false;+ for (int i = 1; i < argc; ++i) {+ const std::string a = argv[i];+ if (a == "--dir" && i + 1 < argc) dir = argv[++i];+ else if (a == "--seconds" && i + 1 < argc) seconds = std::atof(argv[++i]);+ else if (a == "--big") big = true;+ else if (a == "--only-error-path") only_error = true;+ else { std::fprintf(stderr, "usage: ple_reader_stress_test [--dir D] [--seconds S] [--big] [--only-error-path]\n"); return 2; }+ }+ // a file system without direct reads (tmpfs on an older kernel): say so and skip+ {+ const std::string probe = dir + "/ple_stress_probe.bin";+ const Spec s{"probe", 90, 192, 1000, 0};+ if (!make_table(probe, s)) { std::fprintf(stderr, "cannot write in %s\n", dir.c_str()); return 2; }+ ng::PleReader rd;+ std::string err;+ const bool ok = rd.open(probe, s.header, s.n_rows, 4, 0, err, true, s.rb);+ rd.close();+ std::filesystem::remove(probe);+ if (!ok) {+ std::printf("SKIP: direct reads are not possible in %s (%s)\n", dir.c_str(), err.c_str());+ return 77;+ }+ }+ const Spec specs[] = {+ {"iq4nl", 90, 192, 150000, 0}, // the real shard's shape: header 192, 90-byte rows, the table ends the file+ {"iq4nl+tail", 90, 192, 150000, 777}, // bytes after the table+ {"odd phase", 90, 4051, 120000, 0}, // the first row straddles+ {"q5_0", 110, 4000, 100000, 0},+ {"fp8", 160, 0, 90000, 123},+ {"page row", 4096, 192, 3000, 0}, // every row crosses a page boundary+ {"tiny", 1, 5, 30000, 0},+ };+ std::vector<Config> cfgs;+ for (uint32_t inflight : {1u, 2u, 3u, 64u})+ for (uint64_t cache : {0ull, 16ull, 100000ull})+ cfgs.push_back({true, inflight, cache, (inflight + cache) % 2 == 0, inflight == 2, 4});+ cfgs.push_back({false, 1, 16, false, false, 1});+ cfgs.push_back({false, 8, 100000, false, false, 1});+ cfgs.push_back({false, 64, 0, true, false, 1});+ std::printf("ple_reader stress: %zu tables x %zu configurations, %.1f s each (dir %s)\n", sizeof specs / sizeof specs[0],+ cfgs.size(), seconds, dir.c_str());+ uint32_t seed = 1;+ for (const Spec& s : specs) {+ if (only_error) break;+ const std::string path = dir + "/ple_stress_" + std::to_string(s.rb) + "_" + std::to_string(s.header) + ".bin";+ if (!make_table(path, s)) { std::fprintf(stderr, "cannot write %s\n", path.c_str()); return 2; }+ for (const Config& c : cfgs) {+ if (!run_config(dir, s, c, seconds, ++seed)) return 2;+ if (g_fail.load() > 0) break;+ }+ std::filesystem::remove(path);+ if (g_fail.load() > 0) break;+ }+#if !defined(_WIN32)+ if (g_fail.load() == 0) error_does_not_hang(dir);+#endif+ if (big && g_fail.load() == 0) big_offsets(dir);+ std::printf("ple_reader stress: %s\n", g_fail.load() ? "FAILED" : "OK");+ return g_fail.load() ? 1 : 0;+}diff --git a/src/ngram/ple_reader_test.cpp b/src/ngram/ple_reader_test.cppindex f49422c6a..9a4202482 100644--- a/src/ngram/ple_reader_test.cpp+++ b/src/ngram/ple_reader_test.cpp@@ -10,6 +10,7 @@#include "strata/ngram/ple_reader.hpp"#include "strata/platform/direct_file.hpp"+#include <atomic>#include <chrono>#include <cstdio>#include <cstdlib>@@ -325,10 +326,123 @@ void prefetched_keys_are_reused(const std::string& dir) {if (g_fail == 0) std::printf("ple prefetched-key reuse (reordered, duplicated, subset), Direct vs Mmap bytes: OK\n");}+/// #3602: the table the way the engine uses it - one PleTable shared by the prompt path (big gather_batch calls on a+/// thread of their own, as prefill's std::async does), the decode window (prefetch_rows for each token, then+/// gather_batch of the window) and the single-token gather, all at once, with a row cache that evicts all the time+/// and one in-flight slot. Every float must equal what the mapped reader returns for the same rows.+void table_is_consistent_under_concurrent_use(const std::string& dir) {+ const k::PleFormatInfo* fmt = nullptr;+ for (int i = 0; i < k::ple_format_count(); ++i)+ if (std::string(k::ple_formats()[i].name) == "IQ4_NL") fmt = &k::ple_formats()[i];+ if (fmt == nullptr) { CHECK(false, "no IQ4_NL format"); return; }+ const uint32_t rows = 30000;+ const std::string path = write_format_table(dir, *fmt, rows);+ if (path.empty()) { CHECK(false, "cannot write the concurrency table"); return; }+ k::PleTable mapped;+ {+ k::PleIoOptions mio;+ mio.mode = k::PleIo::Mmap;+ std::string err;+ if (!mapped.open(path, err, mio)) { CHECK(false, "mapped open: %s", err.c_str()); return; }+ }+ const size_t tok = (size_t) k::PLE_N_HEADS * k::PLE_HEAD_DIM;+ // rows that straddle a page: row r sits at (header + 90 r), the header is the GGUF's; just take every 7th and every 45th+ auto pick_rows = [&](std::mt19937& rng, size_t n) {+ std::vector<uint32_t> r(n);+ for (auto& x : r) {+ switch (rng() % 6) {+ case 0: x = rows - 1 - rng() % 8; break;+ case 1: x = rng() % 8; break;+ case 2: x = (uint32_t) (rng() % (rows / 45)) * 45u; break;+ case 3: x = 100 + rng() % 12; break; // a hot neighbourhood+ case 4: x = rows + rng() % 3; break; // out of range: zeros in both readers+ default: x = rng() % rows;+ }+ }+ return r;+ };+ for (uint64_t cache : {uint64_t(0), uint64_t(16), uint64_t(4096)})+ for (uint32_t inflight : {1u, 3u, 64u}) {+ k::PleTable t;+ k::PleIoOptions io;+ io.mode = k::PleIo::Direct;+ io.cache_rows = cache;+ io.max_inflight = inflight;+ io.io_thread = true;+ io.keepalive_ms = inflight == 3 ? 1.0 : 0;+ std::string err;+ if (!t.open(path, err, io)) { CHECK(false, "direct open: %s", err.c_str()); return; }+ std::atomic<int> bad{0};+ std::printf(" table under concurrent use: cache %llu rows, inflight %u\n", (unsigned long long) cache, inflight);+ auto same = [&](const std::vector<uint32_t>& r, size_t n_tokens, const float* got, const char* what) {+ std::vector<float> ref(n_tokens * tok);+ std::string e;+ if (!mapped.gather_batch(r.data(), n_tokens, ref.data(), e)) { ++bad; CHECK(false, "mapped reference: %s", e.c_str()); return; }+ // (equal as numbers: a row past the table's end is zero bytes in both readers, but the direct one decodes the+ // zero bytes, which can give -0.0 where the mapped one writes +0.0; the table's bytes are random, so some values are NaN:+ // those are compared by their bits)+ size_t at = 0;+ while (at < ref.size() && (ref[at] == got[at] || std::memcmp(&ref[at], &got[at], sizeof(float)) == 0)) ++at;+ if (at < ref.size()) {+ ++bad;+ CHECK(false, "%s (cache %llu, inflight %u): value %zu of %zu differs from the mapped reader (row %u of %u in range: mapped %g, direct %g)",+ what, (unsigned long long) cache, inflight, at, ref.size(), r[at / k::PLE_HEAD_DIM], rows,+ (double) ref[at], (double) got[at]);+ }+ };+ std::thread prompt([&] { try { // prefill: chunks of tokens, gathered on a thread+ std::mt19937 rng(5);+ for (int i = 0; i < 12 && bad == 0; ++i) {+ static const size_t sizes[4] = {17, 300, 700, 64};+ const size_t nt = sizes[rng() % 4];+ const auto r = pick_rows(rng, nt * k::PLE_N_HEADS);+ std::vector<float> out(nt * tok, -7.0f);+ std::string e;+ if (!t.gather_batch(r.data(), nt, out.data(), e)) { ++bad; CHECK(false, "prompt gather_batch: %s", e.c_str()); return; }+ same(r, nt, out.data(), "prompt gather_batch");+ }+ } catch (const std::exception& e) { ++bad; CHECK(false, "prompt thread: exception %s", e.what()); } });+ std::thread decode([&] { try { // verify windows: prefetch each token's rows, then gather the window+ std::mt19937 rng(6);+ for (int i = 0; i < 300 && bad == 0; ++i) {+ const size_t T = 1 + rng() % 8;+ const auto r = pick_rows(rng, T * k::PLE_N_HEADS);+ for (size_t j = 0; j < T; ++j) t.prefetch_rows(r.data() + j * k::PLE_N_HEADS);+ if (rng() % 4 == 0) { // a prefetch nobody asks for (a draft that was dropped)+ const auto extra = pick_rows(rng, k::PLE_N_HEADS);+ t.prefetch_rows(extra.data());+ }+ std::vector<float> out(T * tok, -7.0f);+ std::string e;+ if (!t.gather_batch(r.data(), T, out.data(), e)) { ++bad; CHECK(false, "window gather_batch: %s", e.c_str()); return; }+ same(r, T, out.data(), "window gather_batch");+ }+ } catch (const std::exception& e) { ++bad; CHECK(false, "decode thread: exception %s", e.what()); } });+ std::thread single([&] { try { // the single-token gather+ std::mt19937 rng(7);+ for (int i = 0; i < 300 && bad == 0; ++i) {+ const auto r = pick_rows(rng, k::PLE_N_HEADS);+ std::vector<float> out(tok, -7.0f);+ t.gather(r.data(), out.data());+ same(r, 1, out.data(), "gather");+ }+ } catch (const std::exception& e) { ++bad; CHECK(false, "single thread: exception %s", e.what()); } });+ prompt.join();+ decode.join();+ single.join();+ t.close();+ }+ mapped.close(); // (Windows cannot delete a file that is still mapped, and remove() throws)+ std::error_code rm_ec;+ std::filesystem::remove(path, rm_ec);+ if (g_fail == 0) std::printf("ple table shared by a prompt thread, a decode window and a single gather, 9 reader configurations, vs the mapped reader: OK\n");+}+void all_formats_round_trip(const std::string& dir) {bf16_widening_is_exact(); // BF16's own claim: a shift, exact for every patternbad_sizes_are_refused(dir);prefetched_keys_are_reused(dir);+ table_is_consistent_under_concurrent_use(dir);CHECK(k::ple_format_count() > 0, "no formats");for (int i = 0; i < k::ple_format_count(); ++i) {const k::PleFormatInfo& f = k::ple_formats()[i];@@ -536,6 +650,7 @@ int real(const std::string& gguf, int n_random, const std::string& tokens_path,} // namespaceint main(int argc, char** argv) {+ std::setvbuf(stdout, nullptr, _IONBF, 0); // (progress lines survive a crash)std::string gguf, dir = std::filesystem::temp_directory_path().string(), tokens;int rows = 20000;uint32_t inflight = 64;diff --git a/sycl/src/ngram/ple_reader.cpp b/sycl/src/ngram/ple_reader.cppindex 4a9df10e6..4cfc919fd 100644--- a/sycl/src/ngram/ple_reader.cpp+++ b/sycl/src/ngram/ple_reader.cpp@@ -442,6 +442,13 @@ PleReader::Ticket PleReader::issue(const uint32_t* rows, size_t n, uint8_t* out_const bool rearm = m.keep_us > 0 && (m.last_issue_us <= 0 || now - m.last_issue_us > m.keep_window_us);m.last_issue_us = now;TicketState& ts = m.tickets[id];+ // After a failed read the reader stays failed (`collect` reports `error` until the next `open`) and its worker has+ // parked (worker_loop). A ticket queued now would never be read and `collect` would wait for it for ever, so it+ // is not queued: it has no jobs, and `collect` returns the error at once.+ if (!m.error.empty()) {+ ts.pending = 0;+ return Ticket{id};+ }std::unordered_map<uint64_t, size_t> by_page; // aligned offset -> index in `jobs`std::vector<Job> jobs;for (size_t i = 0; i < n; ++i) {@@ -510,7 +517,7 @@ bool PleReader::collect(Ticket t, std::string& err) {});it = m.tickets.find(t.id);if (it == m.tickets.end()) { err = "PleReader: unknown ticket"; return false; }- if (!m.error.empty()) { err = m.error; return false; }+ if (!m.error.empty()) { err = m.error; m.tickets.erase(it); return false; }m.stats.wait_us += now_us() - start;if (ple_trace() && it->second.jobs > 0 && it->second.jobs < 200) {const TicketState& ts = it->second;@@ -530,7 +537,7 @@ bool PleReader::collect(Ticket t, std::string& err) {it = m.tickets.find(t.id);if (it == m.tickets.end()) { err = "PleReader: unknown ticket"; return false; }}- if (!m.error.empty()) { err = m.error; return false; }+ if (!m.error.empty()) { err = m.error; m.tickets.erase(it); return false; }m.stats.wait_us += now_us() - start;m.tickets.erase(it);return true;