Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

DeepSeek engine: DS4_COPY_HUGETLB=1 puts the CPU lane's RAM copy on explicit 2 MB pages (MAP_HUGETLB from the kernel pool; the copy's RAM budget adds the pool, which MemAvailable does not count). With DS4_COPY_PIN=1 the pin then keeps 2 MB pages: on a transparent-huge-page arena the registration pins 4 KB pages and the prompt's DMA from it swings between 11.9 and 30.4 s for the same 140.2 GB stream. R9700 12 GB cell, 2.25 pack, 8K, 3 warm prompts: huge-page arena + pin + DS4_RING_MB=3500 + DS4_ATTN_SLAB=2048 -> 694.2 / 693.0 / 691.2 ppts (11.80-11.82 s), first prompt 435.4 (18.8 s), ids identical (5705e1164a); the THP arena with the pin ran 691/573/270 in one load and 665-691 in another. Copy at load 33.3 s for 156.7 GB; the pool is reserved once per boot (sysctl vm.nr_hugepages=78000). Opt-in: default path unchanged; no-pin staged stays 651.

Niko12212dde837d9fd8
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index d49aee4d0..02f33a479 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -4283,7 +4283,11 @@ struct Model::Impl {
}
const uint64_t avail = ram_avail();
// (Windows: the copies are committed memory, within what its commit still has beside the other programs' margin)
- const double can = std::min(avail == ~uint64_t{0} ? 1e300 : std::max(0.0, (double) avail - (double) kRamMargin),
+ // DS4_COPY_HUGETLB: the arena comes from the kernel's huge-page pool, which MemAvailable does not count: add
+ // its free bytes so the full copy is taken when the pool holds it.
+ static const bool huge_arena = std::getenv("DS4_COPY_HUGETLB") != nullptr && std::atoi(std::getenv("DS4_COPY_HUGETLB")) != 0;
+ const double pool = huge_arena ? (double) os::huge_free_bytes() : 0.0;
+ const double can = std::min(avail == ~uint64_t{0} ? 1e300 : std::max(0.0, (double) avail - (double) kRamMargin) + pool,
commit_room());
// (auto: none - copies that all fit were measured no faster than the page cache: the CPU lane's rows are
// compute-bound, ~46 GB/s from either on 2 Xeon sockets, NUMA parts included; a part copied cost decode speed
@@ -4342,13 +4346,18 @@ struct Model::Impl {
if (ns == 0) continue;
const size_t part = cp_lay[(size_t) l.il][3], bytes = (size_t) ns * part;
std::array<uint8_t*, 2> base = {nullptr, nullptr};
+ // DS4_COPY_HUGETLB=1: the arena on explicit 2 MB pages (the pool must be reserved), so a pin of it keeps
+ // 2 MB pages and the prompt's DMA reads stay stable (a THP arena pinned by cudaHostRegister runs in 4 KB
+ // pages: measured 11.9 s one prompt, 30.4 s the next for the same 140 GB)
+ static const bool huge = std::getenv("DS4_COPY_HUGETLB") != nullptr && std::atoi(std::getenv("DS4_COPY_HUGETLB")) != 0;
bool ok = true;
for (int X = 0; X < NH && ok; ++X) {
- base[(size_t) X] = (uint8_t*) node_map(bytes, X, NH);
+ size_t got = bytes;
+ base[(size_t) X] = (uint8_t*) (huge ? os::huge_alloc(bytes, &got) : node_map(bytes, X, NH));
ok = base[(size_t) X] != nullptr;
if (ok) {
ram_chunks.push_back(base[(size_t) X]);
- ram_chunk_bytes.push_back(bytes);
+ ram_chunk_bytes.push_back(got);
}
}
if (!ok) {
diff --git a/src/dsv41/ds4_os.hpp b/src/dsv41/ds4_os.hpp
index 916d993fd..0e48e9d6c 100644
--- a/src/dsv41/ds4_os.hpp
+++ b/src/dsv41/ds4_os.hpp
@@ -398,6 +398,45 @@ inline bool in_ram(const void* p, size_t n) {
#endif
}
+/// Bytes free in the kernel's explicit huge-page pool (HugePages_Free x Hugepagesize); 0 elsewhere or when unreadable.
+/// (MemAvailable does not count a reserved pool, so a budget that must see it - the copy arena on DS4_COPY_HUGETLB -
+/// adds this.)
+inline uint64_t huge_free_bytes() {
+#if defined(_WIN32)
+ return 0;
+#else
+ std::FILE* f = std::fopen("/proc/meminfo", "r");
+ if (f == nullptr) return 0;
+ char line[256];
+ uint64_t pages = 0, size_kb = 0;
+ while (std::fgets(line, sizeof line, f) != nullptr) {
+ unsigned long long v = 0;
+ if (std::sscanf(line, "HugePages_Free: %llu", &v) == 1) pages = v;
+ else if (std::sscanf(line, "Hugepagesize: %llu kB", &v) == 1) size_kb = v;
+ }
+ std::fclose(f);
+ return pages * size_kb * 1024ull;
+#endif
+}
+
+/// Anonymous memory on explicit 2 MB huge pages (MAP_HUGETLB, from the kernel's pool - reserve it first, e.g.
+/// `sysctl vm.nr_hugepages=N`): rounded up to a whole number of huge pages, `*got` its size. nullptr: the pool is empty
+/// or the kernel cannot. (Unlike THP, a pin of this memory keeps 2 MB pages: a DMA from it does not need 4 KB
+/// translations, which is what DS4_COPY_PIN's registered arena wanted.)
+inline void* huge_alloc(size_t bytes, size_t* got = nullptr) {
+#if defined(_WIN32)
+ (void) bytes, (void) got;
+ return nullptr;
+#else
+ const size_t hp = (size_t) 2 << 20;
+ const size_t hb = (bytes + hp - 1) / hp * hp;
+ void* m = ::mmap(nullptr, hb, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0);
+ if (m == MAP_FAILED) return nullptr;
+ if (got != nullptr) *got = hb;
+ return m;
+#endif
+}
+
/// Anonymous memory on NUMA node `node` of `nodes` (node < 0 or nodes < 2: wherever the first touch puts it), on huge
/// pages where the OS gives them; nullptr if it cannot be had.
inline void* anon_alloc(size_t bytes, int node, int nodes) {