Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
qwen35: a penalty for a history token outside the logit row no longer indexes past the row (#3911)
Niko12216c01f85add21
diff --git a/CMakeLists.txt b/CMakeLists.txtindex adfcf6fcb..3fed5ff6a 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -1492,6 +1492,11 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)target_include_directories(ple_residency_test PRIVATE include)add_test(NAME ple_residency_test COMMAND ple_residency_test)+ # 0.2.7 (#3911): the qwen35 host sampler's penalties on a short (draft-vocab) logit row; header-only, CPU+ add_executable(qwen35_sampling_test src/qwen35/sampling_test.cpp)+ target_include_directories(qwen35_sampling_test PRIVATE include)+ add_test(NAME qwen35_sampling_test COMMAND qwen35_sampling_test)+# The GPU test runs ON THE TARGET PC. `--selftest` allocates# and writes real VRAM, so a machine without a device fails this test rather than skipping it silently -# which is correct: an engine that targets one GPU should not report a pass for having no GPU.diff --git a/include/strata/qwen35/sampling.hpp b/include/strata/qwen35/sampling.hppnew file mode 100644index 000000000..8a4a77a66--- /dev/null+++ b/include/strata/qwen35/sampling.hpp@@ -0,0 +1,36 @@+#pragma once+// The qwen35 engine's host sampler settings and the repeat / frequency / presence penalties, header-only so a unit test+// (src/qwen35/sampling_test.cpp) can run them without a GPU.+#include <cstddef>+#include <cstdint>+#include <map>+#include <vector>++namespace strata::qwen35 {++struct Sampling {+ float temperature = 0.f, top_p = 1.f, min_p = 0.f, repeat = 1.f, freq = 0.f, present = 0.f;+ int top_k = 20, last_n = 0;+ unsigned long long seed = 0;+ bool needs_logits() const { return temperature > 0.f || repeat != 1.f || freq != 0.f || present != 0.f; }+};++/// The penalties on the logit row `lg` for the tokens of the last `s.last_n` (all when < 0) history tokens. A row can be+/// shorter than the vocabulary (the MTP draft head picks among the first N token ids only, --draft-vocab N): a history+/// token that is not in the row (id < 0 or >= lg.size()) is not in it and takes no penalty - it used to index past the+/// row (#3911).+inline void apply_penalties(std::vector<float>& lg, const Sampling& s, const std::vector<int32_t>& hist) {+ if (s.last_n != 0 && (s.repeat != 1.f || s.freq != 0.f || s.present != 0.f)) {+ std::map<int, int> cnt;+ const size_t from = s.last_n < 0 || (size_t) s.last_n >= hist.size() ? 0 : hist.size() - (size_t) s.last_n;+ for (size_t i = from; i < hist.size(); ++i) ++cnt[hist[i]];+ for (const auto& [id, c] : cnt) {+ if (id < 0 || (size_t) id >= lg.size()) continue;+ float& v = lg[(size_t) id];+ if (s.repeat != 1.f) v = v > 0 ? v / s.repeat : v * s.repeat;+ v -= (float) c * s.freq + s.present;+ }+ }+}++} // namespace strata::qwen35diff --git a/src/qwen35/q36_engine.cu b/src/qwen35/q36_engine.cuindex 03660de40..6a773837d 100644--- a/src/qwen35/q36_engine.cu+++ b/src/qwen35/q36_engine.cu@@ -26,6 +26,7 @@#include "strata/prefill/gemm.hpp"#include "strata/prefill/moe_mmq.hpp"#include "strata/qwen35/engine.hpp"+#include "strata/qwen35/sampling.hpp"#include "q36_kernels.cuh"#if defined(STRATA_Q36_CPU_POOL)#include "strata/kernels/cpu/native_expert.hpp"@@ -3009,24 +3010,10 @@ bool place_experts(Model& m, Engine& E, double vram_gib, int threads, int hybrid}// ------------------------------------------------------------------------------------------------ sampling-struct Sampling {- float temperature = 0.f, top_p = 1.f, min_p = 0.f, repeat = 1.f, freq = 0.f, present = 0.f;- int top_k = 20, last_n = 0;- unsigned long long seed = 0;- bool needs_logits() const { return temperature > 0.f || repeat != 1.f || freq != 0.f || present != 0.f; }-};+using strata::qwen35::Sampling;int sample(std::vector<float>& lg, const Sampling& s, const std::vector<int32_t>& hist, std::mt19937_64& rng) {- if (s.last_n != 0 && (s.repeat != 1.f || s.freq != 0.f || s.present != 0.f)) {- std::map<int, int> cnt;- const size_t from = s.last_n < 0 || (size_t) s.last_n >= hist.size() ? 0 : hist.size() - (size_t) s.last_n;- for (size_t i = from; i < hist.size(); ++i) ++cnt[hist[i]];- for (const auto& [id, c] : cnt) {- float& v = lg[(size_t) id];- if (s.repeat != 1.f) v = v > 0 ? v / s.repeat : v * s.repeat;- v -= (float) c * s.freq + s.present;- }- }+ strata::qwen35::apply_penalties(lg, s, hist);if (s.temperature <= 0.f) return (int) (std::max_element(lg.begin(), lg.end()) - lg.begin());const int kk = std::max(1, std::min(s.top_k > 0 ? s.top_k : 64, (int) lg.size()));std::vector<int> idx(lg.size());diff --git a/src/qwen35/sampling_test.cpp b/src/qwen35/sampling_test.cppnew file mode 100644index 000000000..4b8e2a590--- /dev/null+++ b/src/qwen35/sampling_test.cpp@@ -0,0 +1,112 @@+// The qwen35 host sampler's penalties (include/strata/qwen35/sampling.hpp), CPU only, no model.+// #3911: a history token outside a short logit row (--draft-vocab N) must not write past the row.+#include "strata/qwen35/sampling.hpp"++#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <map>++using strata::qwen35::Sampling;+using strata::qwen35::apply_penalties;++namespace {+int checks = 0;+void check(bool ok, const char* label) {+ ++checks;+ if (!ok) { std::fprintf(stderr, "FAIL: %s\n", label); std::exit(1); }+}++// The loop as it was before the guard (origin/main of the time), for the in-range comparison.+void old_penalties(std::vector<float>& lg, const Sampling& s, const std::vector<int32_t>& hist) {+ if (s.last_n != 0 && (s.repeat != 1.f || s.freq != 0.f || s.present != 0.f)) {+ std::map<int, int> cnt;+ const size_t from = s.last_n < 0 || (size_t) s.last_n >= hist.size() ? 0 : hist.size() - (size_t) s.last_n;+ for (size_t i = from; i < hist.size(); ++i) ++cnt[hist[i]];+ for (const auto& [id, c] : cnt) {+ float& v = lg[(size_t) id];+ if (s.repeat != 1.f) v = v > 0 ? v / s.repeat : v * s.repeat;+ v -= (float) c * s.freq + s.present;+ }+ }+}++bool same_bits(const std::vector<float>& a, const std::vector<float>& b) {+ return a.size() == b.size() && std::memcmp(a.data(), b.data(), a.size() * sizeof(float)) == 0;+}+} // namespace++int main() {+ Sampling s;+ s.repeat = 1.2f; s.freq = 0.1f; s.present = 0.05f; s.last_n = -1;++ // a short row (a 64-token draft vocabulary) and a history with tokens inside it, past it and negative. The row's+ // storage past its size is filled with a marker first: a write past the row would change the marker.+ {+ std::vector<float> row(256, 7.f);+ row.resize(64);+ for (int i = 0; i < 64; ++i) row[(size_t) i] = (float) i - 20.f;+ const std::vector<float> before(row.begin(), row.end());+ const std::vector<int32_t> hist = {3, 100, 3, 63, 64, 65, 255, 100000, -1, 7};+ apply_penalties(row, s, hist);+ for (int i = 64; i < 256; ++i) check(row.data()[i] == 7.f, "nothing is written past the row");+ check(row.size() == 64, "the row keeps its size");+ // tokens 3 (twice), 63 and 7 are in the row and are penalised; the others are skipped+ std::vector<float> want = before;+ for (int id : {3, 7, 63}) {+ const int c = id == 3 ? 2 : 1;+ float& v = want[(size_t) id];+ v = v > 0 ? v / s.repeat : v * s.repeat;+ v -= (float) c * s.freq + s.present;+ }+ check(same_bits(row, want), "the in-row tokens take the penalty, the rest of the row is untouched");+ }++ // history tokens that are all outside the row: the row does not change+ {+ std::vector<float> row(16, 1.5f);+ const std::vector<float> before = row;+ apply_penalties(row, s, {16, 17, 1000, -5});+ check(same_bits(row, before), "only out-of-row tokens: no change");+ }++ // a row of the whole vocabulary behaves exactly as before the guard, bit for bit, for every window and penalty mix+ {+ const int V = 512;+ std::vector<float> base((size_t) V);+ for (int i = 0; i < V; ++i) base[(size_t) i] = ((float) ((i * 37) % 101) - 50.f) / 7.f;+ std::vector<int32_t> hist;+ for (int i = 0; i < 300; ++i) hist.push_back((i * 53 + 11) % V);+ const float repeats[] = {1.f, 1.1f, 1.5f};+ const float freqs[] = {0.f, 0.2f};+ const float presents[] = {0.f, 0.3f};+ const int lasts[] = {0, -1, 1, 64, 300, 5000};+ for (float r : repeats) for (float f : freqs) for (float p : presents) for (int n : lasts) {+ Sampling t;+ t.repeat = r; t.freq = f; t.present = p; t.last_n = n;+ std::vector<float> a = base, b = base;+ apply_penalties(a, t, hist);+ old_penalties(b, t, hist);+ check(same_bits(a, b), "full-vocabulary row: identical to the unguarded loop");+ }+ }++ // no penalty configured: untouched even with a history+ {+ Sampling t;+ std::vector<float> row(8, 2.f);+ const std::vector<float> before = row;+ apply_penalties(row, t, {1, 2, 99});+ check(same_bits(row, before), "no penalty: no change");+ }++ // an empty row with a history does not crash+ {+ std::vector<float> row;+ apply_penalties(row, s, {0, 1, 2});+ check(row.empty(), "empty row");+ }++ std::printf("qwen35_sampling_test: %d checks passed\n", checks);+ return 0;+}