Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Prefill PLE: opt-in STRATA_PLE_COMPACT=1 uploads the raw IQ4_NL rows and dequantizes them on the device (#3779, #3712)

linxuhao17b08c96339f
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 31d3664b6..69a3fa7c0 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1428,6 +1428,12 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
add_test(NAME ple_parity COMMAND ple_parity --selftest
--in "${STRATA_PLE_FIXTURE_DIR}/ple_in.bin" --out "${STRATA_PLE_FIXTURE_DIR}/ple_out.bin"
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+ # STRATA_PLE_COMPACT: the raw IQ4_NL rows dequantized on the device against the host decode, bit for bit (synthetic, no model)
+ add_executable(ple_iq4nl_dequant_test src/kernels/ple_iq4nl_dequant_test.cpp)
+ target_include_directories(ple_iq4nl_dequant_test PRIVATE ${_strata_gpu_include_directories})
+ target_link_libraries(ple_iq4nl_dequant_test PRIVATE strata_kernels ${_strata_gpu_runtime_target})
+ add_test(NAME ple_iq4nl_dequant_test COMMAND ple_iq4nl_dequant_test)
+ set_tests_properties(ple_iq4nl_dequant_test PROPERTIES SKIP_RETURN_CODE 77)
# Plan v0.3 P7: INT8 KV storage vs a host reference and vs the FP16 path (GPU, synthetic, no model).
add_executable(kv_q8_parity src/kernels/kv_q8_parity.cpp)
diff --git a/include/strata/kernels/ngram.hpp b/include/strata/kernels/ngram.hpp
index d25a1eeec..9954a0f33 100644
--- a/include/strata/kernels/ngram.hpp
+++ b/include/strata/kernels/ngram.hpp
@@ -187,6 +187,10 @@ public:
/// as ONE reader request - page dedupe and sort across the whole batch, the reader's full queue depth. Not
/// while a single-token `issue` is pending. The mapped mode gathers row by row.
bool gather_batch(const uint32_t* rows, size_t n_tokens, float* out, std::string& err);
+ /// STRATA_PLE_COMPACT: the same rows as their raw IQ4_NL bytes (n_tokens x 16 rows of PLE_ROW_BYTES; a row past the
+ /// table: scale 0 and code 8 everywhere, which decode to the +0.0f gather_batch writes) for ple_iq4nl_dequant on the device. False (nothing
+ /// written) unless the table is in memory (--ple-io ram / mmap) and IQ4_NL.
+ bool gather_raw_iq4nl(const uint32_t* rows, size_t n_tokens, uint8_t* out) const;
/// Fault injection (Direct mode): every row read completes no earlier than `us` after issue.
void set_injected_delay_us(double us);
diff --git a/include/strata/kernels/ple.hpp b/include/strata/kernels/ple.hpp
index 98bfc388d..a558e1e27 100644
--- a/include/strata/kernels/ple.hpp
+++ b/include/strata/kernels/ple.hpp
@@ -125,4 +125,8 @@ void ple_history_advance(float* hist, const float* normalized, void* stream);
/// Whether the CUDA path is available. The host hash and the table read work without it; the block does not.
bool ple_block_available();
+/// STRATA_PLE_COMPACT: `rows` raw IQ4_NL PLE rows (PleTable::gather_raw_iq4nl, PLE_ROW_BYTES each) to 160 floats each,
+/// the floats iq4nl_dequant_row writes (d times the code's value, one multiply), on `stream`.
+void ple_iq4nl_dequant(const uint8_t* raw, int64_t rows, float* out, void* stream);
+
} // namespace strata::kernels
diff --git a/metal/src/engine/stubs_0145.cpp b/metal/src/engine/stubs_0145.cpp
index 0c1976f60..791d1df4a 100644
--- a/metal/src/engine/stubs_0145.cpp
+++ b/metal/src/engine/stubs_0145.cpp
@@ -149,3 +149,5 @@ namespace strata::kernels { bool native_q8_0_sm60_pack(void const* p0, int p1, i
namespace strata::kernels { void native_q8_0_sm60_release(void const* p0) { } }
// only reached with pf_wmma (off on Metal)
namespace strata::prefill { void swiglu_split_f16(float const* p0, unsigned short* p1, long long p2, int p3, bool p4, void* p5) { } }
+// STRATA_PLE_COMPACT (opt-in, CUDA / HIP prompt path): prefill.cpp references it; the Metal prompt runs through its window and never reaches it.
+namespace strata::kernels { void ple_iq4nl_dequant(const uint8_t* p0, int64_t p1, float* p2, void* p3) { strata::metal_engine::missing("strata::kernels::ple_iq4nl_dequant"); } }
diff --git a/src/kernels/cuda/ple.cu b/src/kernels/cuda/ple.cu
index f945ef3b8..89c54502e 100644
--- a/src/kernels/cuda/ple.cu
+++ b/src/kernels/cuda/ple.cu
@@ -23,6 +23,7 @@
#include "strata/kernels/native_ple_postops.hpp"
#include <cuda_runtime.h>
+#include <cuda_fp16.h>
#include <cmath>
#include <cstdio>
@@ -387,4 +388,26 @@ void ple_block_projected(const float* projected_key, const float* projected_valu
native_ple_postops(projected_key, hidden, projected_value, hist_rows, w, buffers, stream);
}
+namespace {
+__constant__ int8_t kPleIq4Nl[16] = {-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113};
+// one thread a value: row r, block b (18 bytes: fp16 d, 16 bytes of codes), element j of 32 (j < 16: low nibble of
+// qs[j], else the high nibble of qs[j - 16] - the split halves of iq4nl_dequant_row)
+__global__ void ple_iq4nl_dequant_kernel(const uint8_t* __restrict__ raw, int64_t n, float* __restrict__ out) {
+ const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;
+ if (i >= n) return;
+ const int64_t r = i / PLE_HEAD_DIM;
+ const int e = (int) (i - r * PLE_HEAD_DIM), b = e >> 5, j = e & 31;
+ const uint8_t* blk = raw + r * PLE_ROW_BYTES + b * 18;
+ const float d = __half2float(__ushort_as_half((unsigned short) (blk[0] | (blk[1] << 8))));
+ const uint8_t q = blk[2 + (j & 15)];
+ out[i] = d * (float) kPleIq4Nl[j < 16 ? (q & 15) : (q >> 4)];
+}
+} // namespace
+
+void ple_iq4nl_dequant(const uint8_t* raw, int64_t rows, float* out, void* stream) {
+ const int64_t n = rows * PLE_HEAD_DIM;
+ if (n <= 0) return;
+ ple_iq4nl_dequant_kernel<<<(unsigned) ((n + 255) / 256), 256, 0, (cudaStream_t) stream>>>(raw, n, out);
+}
+
} // namespace strata::kernels
diff --git a/src/kernels/ngram.cpp b/src/kernels/ngram.cpp
index c32dc786a..7b8d844e7 100644
--- a/src/kernels/ngram.cpp
+++ b/src/kernels/ngram.cpp
@@ -618,6 +618,26 @@ bool PleTable::gather_batch(const uint32_t* rows, size_t n_tokens, float* out, s
return true;
}
+bool PleTable::gather_raw_iq4nl(const uint32_t* rows, size_t n_tokens, uint8_t* out) const {
+ std::lock_guard<std::recursive_mutex> lk(impl_->mu);
+ if (impl_->data == nullptr || impl_->fmt->id != PleFormat::IQ4_NL || impl_->rb != PLE_ROW_BYTES) return false;
+ const size_t n = n_tokens * (size_t) PLE_N_HEADS;
+ for (size_t i = 0; i < n; ++i) {
+ uint8_t* o = out + i * PLE_ROW_BYTES;
+ if (rows[i] >= impl_->n_rows) {
+ // a row past the table reads as +0.0f (read_row): scale 0 and code 8 (value +1) in every nibble decode to +0.0f
+ // on the device, where all-zero bytes would decode to -0.0f (scale 0 times code 0's value -127)
+ for (int b = 0; b < PLE_HEAD_DIM / 32; ++b) {
+ uint8_t* blk = o + (size_t) b * 18;
+ std::memset(blk, 0, 2);
+ std::memset(blk + 2, 0x88, 16);
+ }
+ } else std::memcpy(o, impl_->data + (size_t) rows[i] * PLE_ROW_BYTES, PLE_ROW_BYTES);
+ }
+ impl_->bytes_read += (uint64_t) n * PLE_ROW_BYTES;
+ return true;
+}
+
void PleTable::set_injected_delay_us(double us) { impl_->reader.set_injected_delay_us(us); }
std::string PleTable::io_report() const {
diff --git a/src/kernels/ple_iq4nl_dequant_test.cpp b/src/kernels/ple_iq4nl_dequant_test.cpp
new file mode 100644
index 000000000..e5312cf92
--- /dev/null
+++ b/src/kernels/ple_iq4nl_dequant_test.cpp
@@ -0,0 +1,101 @@
+// src/kernels/ple_iq4nl_dequant_test.cpp - STRATA_PLE_COMPACT: ple_iq4nl_dequant (raw IQ4_NL PLE rows to floats on the
+// device) against iq4nl_dequant_row (the host decode the prompt path used before), bit for bit. Synthetic bytes, no model.
+//
+// - every one of the 65,536 fp16 scale patterns (infinities and NaNs included: those are compared as "both NaN") in
+// every block position, with random codes;
+// - rows of random bytes;
+// - the row the table gives for an index past its end (scale 0, code 8 in every nibble: +0.0f, what PleTable::read_row
+// writes for such a row);
+// - an odd row count (not a multiple of the 256-thread block).
+// Exit 77 without a CUDA device.
+#include "strata/kernels/ngram.hpp"
+#include "strata/kernels/ple.hpp"
+
+#include <cuda_runtime.h>
+
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <random>
+#include <vector>
+
+namespace {
+using namespace strata::kernels;
+
+int check(const std::vector<uint8_t>& raw, int64_t rows, const char* what) {
+ float* dout = nullptr;
+ uint8_t* draw = nullptr;
+ if (cudaMalloc((void**) &dout, (size_t) rows * PLE_HEAD_DIM * 4) != cudaSuccess ||
+ cudaMalloc((void**) &draw, raw.size()) != cudaSuccess ||
+ cudaMemcpy(draw, raw.data(), raw.size(), cudaMemcpyHostToDevice) != cudaSuccess) {
+ std::printf("CUDA setup failed for %s\n", what);
+ return 1;
+ }
+ cudaMemset(dout, 0xff, (size_t) rows * PLE_HEAD_DIM * 4);
+ ple_iq4nl_dequant(draw, rows, dout, nullptr);
+ if (cudaDeviceSynchronize() != cudaSuccess) { std::printf("CUDA error for %s: %s\n", what, cudaGetErrorString(cudaGetLastError())); return 1; }
+ std::vector<float> got((size_t) rows * PLE_HEAD_DIM), want(got.size());
+ cudaMemcpy(got.data(), dout, got.size() * 4, cudaMemcpyDeviceToHost);
+ cudaFree(dout);
+ cudaFree(draw);
+ for (int64_t r = 0; r < rows; ++r) iq4nl_dequant_row(raw.data() + (size_t) r * PLE_ROW_BYTES, want.data() + (size_t) r * PLE_HEAD_DIM);
+ long long diff = 0, first = -1;
+ for (size_t i = 0; i < got.size(); ++i) {
+ uint32_t a, b;
+ std::memcpy(&a, &got[i], 4);
+ std::memcpy(&b, &want[i], 4);
+ if (std::isnan(got[i]) && std::isnan(want[i])) continue; // a NaN scale: any payload is a NaN
+ if (a != b) { if (first < 0) first = (long long) i; ++diff; }
+ }
+ std::printf(" %-52s %lld rows: %s\n", what, (long long) rows, diff ? "*** DIFFERS ***" : "bitwise equal");
+ if (diff) std::printf(" %lld values differ, first at %lld\n", diff, first);
+ return diff ? 1 : 0;
+}
+} // namespace
+
+int main() {
+ int n_dev = 0;
+ if (cudaGetDeviceCount(&n_dev) != cudaSuccess || n_dev < 1) { std::printf("no CUDA device (skipped)\n"); return 77; }
+ int bad = 0;
+ std::mt19937 rng(20261026u);
+ { // all fp16 patterns: 65,536 scales over 5 blocks a row = 13,108 rows (the last row pads with 0)
+ const int64_t rows = (65536 + 4) / 5;
+ std::vector<uint8_t> raw((size_t) rows * PLE_ROW_BYTES);
+ for (auto& b : raw) b = (uint8_t) (rng() & 0xff);
+ uint32_t d = 0;
+ for (int64_t r = 0; r < rows; ++r)
+ for (int b = 0; b < PLE_HEAD_DIM / 32; ++b, ++d) {
+ const uint16_t h = (uint16_t) (d & 0xffff);
+ std::memcpy(&raw[(size_t) r * PLE_ROW_BYTES + (size_t) b * 18], &h, 2);
+ }
+ bad += check(raw, rows, "every fp16 scale pattern, random codes");
+ }
+ { // random rows
+ const int64_t rows = 4096 + 37;
+ std::vector<uint8_t> raw((size_t) rows * PLE_ROW_BYTES);
+ for (auto& b : raw) b = (uint8_t) (rng() & 0xff);
+ bad += check(raw, rows, "random bytes (odd row count)");
+ }
+ { // the out-of-table row of PleTable::gather_raw_iq4nl
+ const int64_t rows = 16;
+ std::vector<uint8_t> raw((size_t) rows * PLE_ROW_BYTES);
+ for (int64_t r = 0; r < rows; ++r)
+ for (int b = 0; b < PLE_HEAD_DIM / 32; ++b) {
+ uint8_t* blk = &raw[(size_t) r * PLE_ROW_BYTES + (size_t) b * 18];
+ std::memset(blk, 0, 2);
+ std::memset(blk + 2, 0x88, 16);
+ }
+ bad += check(raw, rows, "out-of-table row (scale 0, code 8)");
+ // and it is +0.0f, not -0.0f, on both sides
+ std::vector<float> want(PLE_HEAD_DIM);
+ iq4nl_dequant_row(raw.data(), want.data());
+ for (float v : want) {
+ uint32_t a;
+ std::memcpy(&a, &v, 4);
+ if (a != 0) { std::printf(" *** the out-of-table row does not decode to +0.0f\n"); ++bad; break; }
+ }
+ }
+ std::printf("ple_iq4nl_dequant_test: %s\n", bad ? "FAILED" : "OK");
+ return bad ? 1 : 0;
+}
diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cpp
index a96c2a03d..a101503f0 100644
--- a/src/prefill/prefill.cpp
+++ b/src/prefill/prefill.cpp
@@ -132,6 +132,14 @@ bool prompt_run(const int64_t* tokens, int64_t n, int64_t pos0, const float* con
} // namespace strata::metal_engine
#endif
+namespace {
+// STRATA_PLE_COMPACT=1 (opt-in): the prompt chunk's PLE rows go up raw and are dequantized on the device (IQ4_NL tables
+// held in memory); the same floats as the host dequant, 7x fewer bytes over PCIe
+bool ple_compact_on() {
+ static const bool on = [] { const char* v = std::getenv("STRATA_PLE_COMPACT"); return v != nullptr && v[0] == '1'; }();
+ return on;
+}
+} // namespace
namespace strata::prefill {
namespace {
@@ -987,6 +995,8 @@ struct Prefill::Impl {
float* ple_emb_host[2] = {}; // pinned, double-buffered: the next chunk's rows are read while this
cudaEvent_t ple_copied[2] = {}; // one runs; the event marks that buffer's upload done
std::vector<uint32_t> ple_rows[2];
+ bool ple_compact[2] = {false, false}; // STRATA_PLE_COMPACT: the pinned buffer holds raw IQ4_NL rows
+ uint8_t* ple_raw_dev = nullptr; // their device copy (cudaMalloc'd, the chunk's 16 rows a token)
float* ple_norm = nullptr;
uint8_t* region = nullptr; // the attention/MoE scratch region (idle while the PLE block runs)
uint64_t region_bytes = 0;
@@ -1097,6 +1107,7 @@ void Prefill::release() {
if (e) cudaEventDestroy(e);
if (impl_->ple_copied[b]) cudaEventDestroy(impl_->ple_copied[b]);
if (impl_->ple_emb_host[b] && impl_->ple_pageable[b].empty()) cudaFreeHost(impl_->ple_emb_host[b]);
+ if (b == 0 && impl_->ple_raw_dev) { (void) cudaFree(impl_->ple_raw_dev); impl_->ple_raw_dev = nullptr; }
}
if (impl_->copy) cudaStreamDestroy(impl_->copy);
if (impl_->kv_copy) cudaStreamDestroy(impl_->kv_copy);
@@ -1525,6 +1536,11 @@ bool Prefill::init(const core::WeightTable& wt, const core::ModelGeometry& g, co
if (!m.ple_copied[b] && cudaEventCreateWithFlags(&m.ple_copied[b], cudaEventDisableTiming) != cudaSuccess)
ok = false;
m.ple_rows[b].resize(T * strata::kernels::PLE_N_HEADS);
+ if (ple_compact_on() && m.ple_raw_dev == nullptr &&
+ cudaMalloc((void**) &m.ple_raw_dev, (size_t) T * strata::kernels::PLE_N_HEADS * strata::kernels::PLE_ROW_BYTES) != cudaSuccess) {
+ m.ple_raw_dev = nullptr;
+ (void) cudaGetLastError();
+ }
}
if (ss.qsa_states[ss.qsa_primary()].kv_mode == 1) { // KV streaming: the staging pool's identity page table
const int64_t pages = ss.qsa_states[ss.qsa_primary()].n_pages;
@@ -2839,6 +2855,14 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
pv[0] = pv[1];
pv[1] = tok;
}
+ // STRATA_PLE_COMPACT=1: the raw rows (1,440 bytes a token instead of 10,240 floats' bytes) into the same pinned
+ // buffer; ple_land uploads them and dequantizes on the device
+ if (ple_compact_on() && m.ple_pageable[buf].empty() && m.ple_raw_dev != nullptr &&
+ ss.ple.table->gather_raw_iq4nl(m.ple_rows[buf].data(), (size_t) T, (uint8_t*) m.ple_emb_host[buf])) {
+ m.ple_compact[buf] = true;
+ return true;
+ }
+ m.ple_compact[buf] = false;
return ss.ple.table->gather_batch(m.ple_rows[buf].data(), (size_t) T, m.ple_emb_host[buf], e);
};
std::string ple_next_err;
@@ -3002,7 +3026,10 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
return false;
}
// #1996: the pageable fallback (no RAM to pin) goes up through a pinned slot; the pinned buffers as before
- if ((m.ple_pageable[ple_buf].empty()
+ const size_t raw_bytes = (size_t) T * strata::kernels::PLE_N_HEADS * strata::kernels::PLE_ROW_BYTES;
+ if ((m.ple_compact[ple_buf]
+ ? cudaMemcpyAsync(m.ple_raw_dev, m.ple_emb_host[ple_buf], raw_bytes, cudaMemcpyHostToDevice, m.cs)
+ : m.ple_pageable[ple_buf].empty()
? cudaMemcpyAsync(m.ple_emb, m.ple_emb_host[ple_buf], (size_t) T * N * 4, cudaMemcpyHostToDevice, m.cs)
: m.up_ring.h2d(m.ple_emb, m.ple_emb_host[ple_buf], (size_t) T * N * 4, m.cs)) !=
cudaSuccess ||
@@ -3010,6 +3037,8 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
err = std::string("prefill: the PLE rows' upload failed: ") + cudaGetErrorString(cudaGetLastError());
return false;
}
+ if (m.ple_compact[ple_buf])
+ strata::kernels::ple_iq4nl_dequant(m.ple_raw_dev, T * strata::kernels::PLE_N_HEADS, m.ple_emb, m.cs);
if (c0 + C < n) {
// the other buffer's upload (a chunk ago) is done before the SSD thread refills it
if (cudaEventSynchronize(m.ple_copied[ple_buf ^ 1]) != cudaSuccess) {