Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek engine on an integrated NVIDIA GPU (DGX Spark): arm64 build and placement from MemAvailable
Niko12214e06c84b9159
diff --git a/src/dsv41/ds4_gg_cpu.cpp b/src/dsv41/ds4_gg_cpu.cppindex cede307a9..78853b12c 100644--- a/src/dsv41/ds4_gg_cpu.cpp+++ b/src/dsv41/ds4_gg_cpu.cpp@@ -169,7 +169,7 @@ static int amd_family() {if (!__get_cpuid(1, &a, &b, &c, &d)) return 0;const unsigned eax = a;#else- return 0;+ const unsigned eax = 0; // not x86 (arm64, a DGX Spark): no AMD family, the base family below reads 0#endifconst unsigned base = (eax >> 8) & 0xF;return (int) (base == 0xF ? base + ((eax >> 20) & 0xFF) : base);diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 91d29a266..7ff914fd4 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -203,6 +203,28 @@ uint64_t ram_avail() {return avail;}+// the device memory the experts' placement may count on: cudaMemGetInfo's free, but on an integrated NVIDIA GPU on Linux+// (a DGX Spark's GB10, one memory with the CPU) what the kernel can still give back, MemAvailable less+// DS4_UMA_HEADROOM_GIB (6): cudaMemGetInfo counts only MemFree there, and the model's files just read fill the page cache+// (measured on a GB10: 55 GB "free" with 115 GB available). Not on HIP: a Strix Halo's GPU memory is its carve-out plus+// the GTT pool, which hipMemGetInfo already counts and MemAvailable may exceed+size_t uma_free(int dev, size_t fr) {+#if !defined(_WIN32) && !defined(STRATA_USE_HIP)+ int integrated = 0;+ if (cudaDeviceGetAttribute(&integrated, cudaDevAttrIntegrated, dev) != cudaSuccess) {+ (void) cudaGetLastError();+ return fr;+ }+ if (!integrated) return fr;+ const char* h = std::getenv("DS4_UMA_HEADROOM_GIB");+ const uint64_t head = (uint64_t) (h ? std::atof(h) : 6.0) << 30, avail = ram_avail();+ if (avail != ~uint64_t{0} && avail > head && avail - head > fr) return (size_t) (avail - head);+#else+ (void) dev;+#endif+ return fr;+}+float bf16_to_f32(uint16_t b) {uint32_t u = (uint32_t) b << 16;float f;@@ -1980,6 +2002,7 @@ struct Model::Impl {return false;}CK(cudaMemGetInfo(&d.free0, &d.total));+ d.free0 = uma_free(d.id, d.free0);}// the groups' width: o.tp, less while the experts' blocks do not split in it (a member takes n_ff / tp of each// expert: the down's inputs in whole blocks - a GGUF's 256-value types split n_ff 2304 in 1, 3 or 9 parts)@@ -2742,6 +2765,7 @@ struct Model::Impl {cudaSetDevice(d.id);size_t fr = 0, tot = 0;CK(cudaMemGetInfo(&fr, &tot));+ fr = uma_free(d.id, fr);room += std::max(0.0, (double) fr - (double) ((size_t) o.reserve_mib << 20)) * o.vram_experts;}for (const Layer& l : L) all += (double) l.blob * l.n_exp;@@ -4094,6 +4118,7 @@ struct Model::Impl {cudaSetDevice(devs[(size_t) m].id);size_t fr = 0, tot = 0;CK(cudaMemGetInfo(&fr, &tot));+ fr = uma_free(devs[(size_t) m].id, fr);// (an integrated GPU, Windows: its memory is the system's. What it allocates after the experts - the long// prompt's buffers, the GEMM workspaces - spills into RAM when the experts took every byte of the GPU's// address space, and on a 32K prompt that ran the machine out of RAM. So the experts leave room for them,@@ -5671,6 +5696,7 @@ struct Model::Impl {img_tok = tok;CK(cudaSetDevice(d.id));CK(cudaMemGetInfo(&d.free0, &d.total)); // (the split sees what the tower left)+ d.free0 = uma_free(d.id, d.free0);if (o.verbose)std::fprintf(stderr, "ds4: vision tower (%d blocks) on CUDA%d: %.0f MiB, up to %d patches, %.1f s\n",vg.layers, d.id, (double) vt.device_bytes() / (1 << 20), vt.max_patches(),diff --git a/src/dsv41/vendor/exl3_cpu.cpp b/src/dsv41/vendor/exl3_cpu.cppindex e87efee8b..880035e43 100644--- a/src/dsv41/vendor/exl3_cpu.cpp+++ b/src/dsv41/vendor/exl3_cpu.cpp@@ -140,6 +140,9 @@ void band_scalar(const Exl3Part& m, const float* xh, int tc0, int ks0, int ks1,}}+// band_gemm's chunk of inputs (see below); outside the x86 section, as exl3_cpu_gemm_scratch() sizes by it on any CPU+constexpr int kGemmK = 256;+// ---------------------------------------------------------------- AVX-512#if defined(EXL3_X86)template <int K>@@ -319,8 +322,7 @@ EXL3_AVX512 void rows_avx512_n(const Exl3Part& m, const float* const* xh, int ks// band_gemm: a band's byte-sum products for many activations - each chunk of kGemmK inputs decoded ONCE into F32// (wc, in L2), then multiplied as a GEMM: tiles of TB activations x 64 columns, the sums in registers, two FMAs a// load. The prompt path's CPU rows, where an expert serves tens to hundreds of tokens: one decode (~6 instructions a-// 16-weight row) for all of them instead of one per kExl3CpuMaxN.-constexpr int kGemmK = 256;+// 16-weight row) for all of them instead of one per kExl3CpuMaxN. (kGemmK is defined above the AVX-512 section.)template <int K>EXL3_AVX512 void decode_chunk(const Exl3Part& m, int tc0, int ks0, int ks1, float* wc /* (ks1 - ks0) 16 x 128 */) {