Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek engine: DS4_COPY_HUGETLB=1 puts the CPU lane's RAM copy on explicit 2 MB pages (MAP_HUGETLB from the kernel pool; the copy's RAM budget adds the pool, which MemAvailable does not count). With DS4_COPY_PIN=1 the pin then keeps 2 MB pages: on a transparent-huge-page arena the registration pins 4 KB pages and the prompt's DMA from it swings between 11.9 and 30.4 s for the same 140.2 GB stream. R9700 12 GB cell, 2.25 pack, 8K, 3 warm prompts: huge-page arena + pin + DS4_RING_MB=3500 + DS4_ATTN_SLAB=2048 -> 694.2 / 693.0 / 691.2 ppts (11.80-11.82 s), first prompt 435.4 (18.8 s), ids identical (5705e1164a); the THP arena with the pin ran 691/573/270 in one load and 665-691 in another. Copy at load 33.3 s for 156.7 GB; the pool is reserved once per boot (sysctl vm.nr_hugepages=78000). Opt-in: default path unchanged; no-pin staged stays 651.
Niko12212dde837d9fd8
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex d49aee4d0..02f33a479 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -4283,7 +4283,11 @@ struct Model::Impl {}const uint64_t avail = ram_avail();// (Windows: the copies are committed memory, within what its commit still has beside the other programs' margin)- const double can = std::min(avail == ~uint64_t{0} ? 1e300 : std::max(0.0, (double) avail - (double) kRamMargin),+ // DS4_COPY_HUGETLB: the arena comes from the kernel's huge-page pool, which MemAvailable does not count: add+ // its free bytes so the full copy is taken when the pool holds it.+ static const bool huge_arena = std::getenv("DS4_COPY_HUGETLB") != nullptr && std::atoi(std::getenv("DS4_COPY_HUGETLB")) != 0;+ const double pool = huge_arena ? (double) os::huge_free_bytes() : 0.0;+ const double can = std::min(avail == ~uint64_t{0} ? 1e300 : std::max(0.0, (double) avail - (double) kRamMargin) + pool,commit_room());// (auto: none - copies that all fit were measured no faster than the page cache: the CPU lane's rows are// compute-bound, ~46 GB/s from either on 2 Xeon sockets, NUMA parts included; a part copied cost decode speed@@ -4342,13 +4346,18 @@ struct Model::Impl {if (ns == 0) continue;const size_t part = cp_lay[(size_t) l.il][3], bytes = (size_t) ns * part;std::array<uint8_t*, 2> base = {nullptr, nullptr};+ // DS4_COPY_HUGETLB=1: the arena on explicit 2 MB pages (the pool must be reserved), so a pin of it keeps+ // 2 MB pages and the prompt's DMA reads stay stable (a THP arena pinned by cudaHostRegister runs in 4 KB+ // pages: measured 11.9 s one prompt, 30.4 s the next for the same 140 GB)+ static const bool huge = std::getenv("DS4_COPY_HUGETLB") != nullptr && std::atoi(std::getenv("DS4_COPY_HUGETLB")) != 0;bool ok = true;for (int X = 0; X < NH && ok; ++X) {- base[(size_t) X] = (uint8_t*) node_map(bytes, X, NH);+ size_t got = bytes;+ base[(size_t) X] = (uint8_t*) (huge ? os::huge_alloc(bytes, &got) : node_map(bytes, X, NH));ok = base[(size_t) X] != nullptr;if (ok) {ram_chunks.push_back(base[(size_t) X]);- ram_chunk_bytes.push_back(bytes);+ ram_chunk_bytes.push_back(got);}}if (!ok) {diff --git a/src/dsv41/ds4_os.hpp b/src/dsv41/ds4_os.hppindex 916d993fd..0e48e9d6c 100644--- a/src/dsv41/ds4_os.hpp+++ b/src/dsv41/ds4_os.hpp@@ -398,6 +398,45 @@ inline bool in_ram(const void* p, size_t n) {#endif}+/// Bytes free in the kernel's explicit huge-page pool (HugePages_Free x Hugepagesize); 0 elsewhere or when unreadable.+/// (MemAvailable does not count a reserved pool, so a budget that must see it - the copy arena on DS4_COPY_HUGETLB -+/// adds this.)+inline uint64_t huge_free_bytes() {+#if defined(_WIN32)+ return 0;+#else+ std::FILE* f = std::fopen("/proc/meminfo", "r");+ if (f == nullptr) return 0;+ char line[256];+ uint64_t pages = 0, size_kb = 0;+ while (std::fgets(line, sizeof line, f) != nullptr) {+ unsigned long long v = 0;+ if (std::sscanf(line, "HugePages_Free: %llu", &v) == 1) pages = v;+ else if (std::sscanf(line, "Hugepagesize: %llu kB", &v) == 1) size_kb = v;+ }+ std::fclose(f);+ return pages * size_kb * 1024ull;+#endif+}++/// Anonymous memory on explicit 2 MB huge pages (MAP_HUGETLB, from the kernel's pool - reserve it first, e.g.+/// `sysctl vm.nr_hugepages=N`): rounded up to a whole number of huge pages, `*got` its size. nullptr: the pool is empty+/// or the kernel cannot. (Unlike THP, a pin of this memory keeps 2 MB pages: a DMA from it does not need 4 KB+/// translations, which is what DS4_COPY_PIN's registered arena wanted.)+inline void* huge_alloc(size_t bytes, size_t* got = nullptr) {+#if defined(_WIN32)+ (void) bytes, (void) got;+ return nullptr;+#else+ const size_t hp = (size_t) 2 << 20;+ const size_t hb = (bytes + hp - 1) / hp * hp;+ void* m = ::mmap(nullptr, hb, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0);+ if (m == MAP_FAILED) return nullptr;+ if (got != nullptr) *got = hb;+ return m;+#endif+}+/// Anonymous memory on NUMA node `node` of `nodes` (node < 0 or nodes < 2: wherever the first touch puts it), on huge/// pages where the OS gives them; nullptr if it cannot be had.inline void* anon_alloc(size_t bytes, int node, int nodes) {