Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Native head: opt-in STRATA_EMBED_VRAM=1 keeps the token-embedding table in VRAM (#3779, #3712)

linxuhao884e37dfdfd9
diff --git a/include/strata/core/native_head.hpp b/include/strata/core/native_head.hpp
index 501f6ec27..29fe9d376 100644
--- a/include/strata/core/native_head.hpp
+++ b/include/strata/core/native_head.hpp
@@ -43,7 +43,8 @@ private:
/// Plan v0.3 P6: `token_embd.weight` in its GGUF form (the IQ model files), in mapped pinned host memory: a row
/// is read over PCIe per token, so the table costs no VRAM. `embed_row`, the verify window and the MTP drafter
-/// use it instead of the canonical table when it is set (`set_native_embed`).
+/// use it instead of the canonical table when it is set (`set_native_embed`). `STRATA_EMBED_VRAM=1` places the
+/// same bytes in VRAM instead (slow PCIe links); the gathers read `dev_` either way.
class NativeEmbed {
public:
NativeEmbed() = default;
@@ -57,6 +58,8 @@ public:
void gather_one(int64_t token, float* out, void* stream) const;
uint64_t bytes() const { return bytes_; }
int type() const { return type_; }
+ /// The table was placed in VRAM (STRATA_EMBED_VRAM=1, or the pinned / alias fallbacks) rather than mapped.
+ bool in_vram() const { return dev_ != nullptr && host_ == nullptr; }
private:
void* host_ = nullptr;
diff --git a/src/core/native_head.cpp b/src/core/native_head.cpp
index a1fb19441..f9e757285 100644
--- a/src/core/native_head.cpp
+++ b/src/core/native_head.cpp
@@ -7,6 +7,7 @@
#include <cuda_runtime.h>
#include <climits>
#include <cstdio>
+#include <cstdlib>
#include <cstring>
#include <exception>
@@ -146,6 +147,34 @@ bool NativeEmbed::load(const std::vector<std::string>& shards, int64_t n_embd, i
return false;
}
strata::platform::advise_willneed(gguf.tensor_data(*t), bytes_); // copied out of the mapping below
+ // STRATA_EMBED_VRAM=1: the table in the current device's VRAM instead of mapped host memory. The same
+ // bytes, read by the same gather kernels through dev_ (bitwise the same rows); what changes is where a
+ // prompt's scattered row reads go: over a PCIe Gen2 x4 link they cost more than the table's size in VRAM
+ // is worth. Taken only with the table plus 512 MiB free; otherwise, or when the upload fails, the mapped
+ // placement below is used as before (and the fallback is said).
+ if (const char* ev = std::getenv("STRATA_EMBED_VRAM"); ev != nullptr && ev[0] == '1') {
+ size_t free_b = 0, total_b = 0;
+ const bool have_free = cudaMemGetInfo(&free_b, &total_b) == cudaSuccess;
+ void* d = nullptr;
+ if (have_free && (uint64_t) free_b > bytes_ + ((uint64_t) 512 << 20) &&
+ cudaMalloc(&d, bytes_) == cudaSuccess) {
+ if (cudaMemcpy(d, gguf.tensor_data(*t), bytes_, cudaMemcpyHostToDevice) == cudaSuccess) {
+ std::fprintf(stderr, "strata: native embedding: STRATA_EMBED_VRAM=1: %llu MiB placed in VRAM "
+ "(%llu MiB were free)\n",
+ (unsigned long long) (bytes_ >> 20), (unsigned long long) (free_b >> 20));
+ dev_ = d; // host_ stays null: the destructor frees dev_
+ type_ = (int) t->type;
+ n_embd_ = n_embd;
+ n_vocab_ = n_vocab;
+ return true;
+ }
+ (void) cudaFree(d);
+ }
+ (void) cudaGetLastError();
+ std::fprintf(stderr, "strata: native embedding: STRATA_EMBED_VRAM=1 not taken (%llu MiB free, the table "
+ "needs %llu MiB plus 512 MiB): kept in mapped host memory\n",
+ (unsigned long long) (have_free ? free_b >> 20 : 0), (unsigned long long) (bytes_ >> 20));
+ }
cudaError_t host_err = cudaHostAlloc(&host_, bytes_, cudaHostAllocMapped | cudaHostAllocPortable);
if (host_err != cudaSuccess) {
// Under WSL2 the driver's pinned/mapped host budget (~1 GiB) can be spent by the GPU contexts
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index 54d221cd4..187df09a0 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -3848,9 +3848,11 @@ int main(int argc, char** argv) {
}
strata::core::set_native_embed(&native_embed);
std::fprintf(stderr, "strata generate: native pack: %s experts (largest blob %.2f MB), token embedding "
- "%s in mapped host memory (%.0f MiB, %.1f s)\n",
+ "%s %s (%.0f MiB, %.1f s)\n",
o.pack.c_str(), (double) strata::kernels::cpu::expert_layout().max_blob / 1e6,
- strata::ggml_type_name((uint32_t) native_embed.type()), (double) native_embed.bytes() / 1048576.0,
+ strata::ggml_type_name((uint32_t) native_embed.type()),
+ native_embed.in_vram() ? "in VRAM" : "in mapped host memory",
+ (double) native_embed.bytes() / 1048576.0,
std::chrono::duration<double>(std::chrono::steady_clock::now() - embed_t0).count());
}
// Plan v0.3 P1: tensors served in native form are not also loaded in canonical form (~2.7 GB of VRAM back