Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

qwen35: a penalty for a history token outside the logit row no longer indexes past the row (#3911)

Niko12216c01f85add21
diff --git a/CMakeLists.txt b/CMakeLists.txt
index adfcf6fcb..3fed5ff6a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1492,6 +1492,11 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
target_include_directories(ple_residency_test PRIVATE include)
add_test(NAME ple_residency_test COMMAND ple_residency_test)
+ # 0.2.7 (#3911): the qwen35 host sampler's penalties on a short (draft-vocab) logit row; header-only, CPU
+ add_executable(qwen35_sampling_test src/qwen35/sampling_test.cpp)
+ target_include_directories(qwen35_sampling_test PRIVATE include)
+ add_test(NAME qwen35_sampling_test COMMAND qwen35_sampling_test)
+
# The GPU test runs ON THE TARGET PC. `--selftest` allocates
# and writes real VRAM, so a machine without a device fails this test rather than skipping it silently -
# which is correct: an engine that targets one GPU should not report a pass for having no GPU.
diff --git a/include/strata/qwen35/sampling.hpp b/include/strata/qwen35/sampling.hpp
new file mode 100644
index 000000000..8a4a77a66
--- /dev/null
+++ b/include/strata/qwen35/sampling.hpp
@@ -0,0 +1,36 @@
+#pragma once
+// The qwen35 engine's host sampler settings and the repeat / frequency / presence penalties, header-only so a unit test
+// (src/qwen35/sampling_test.cpp) can run them without a GPU.
+#include <cstddef>
+#include <cstdint>
+#include <map>
+#include <vector>
+
+namespace strata::qwen35 {
+
+struct Sampling {
+ float temperature = 0.f, top_p = 1.f, min_p = 0.f, repeat = 1.f, freq = 0.f, present = 0.f;
+ int top_k = 20, last_n = 0;
+ unsigned long long seed = 0;
+ bool needs_logits() const { return temperature > 0.f || repeat != 1.f || freq != 0.f || present != 0.f; }
+};
+
+/// The penalties on the logit row `lg` for the tokens of the last `s.last_n` (all when < 0) history tokens. A row can be
+/// shorter than the vocabulary (the MTP draft head picks among the first N token ids only, --draft-vocab N): a history
+/// token that is not in the row (id < 0 or >= lg.size()) is not in it and takes no penalty - it used to index past the
+/// row (#3911).
+inline void apply_penalties(std::vector<float>& lg, const Sampling& s, const std::vector<int32_t>& hist) {
+ if (s.last_n != 0 && (s.repeat != 1.f || s.freq != 0.f || s.present != 0.f)) {
+ std::map<int, int> cnt;
+ const size_t from = s.last_n < 0 || (size_t) s.last_n >= hist.size() ? 0 : hist.size() - (size_t) s.last_n;
+ for (size_t i = from; i < hist.size(); ++i) ++cnt[hist[i]];
+ for (const auto& [id, c] : cnt) {
+ if (id < 0 || (size_t) id >= lg.size()) continue;
+ float& v = lg[(size_t) id];
+ if (s.repeat != 1.f) v = v > 0 ? v / s.repeat : v * s.repeat;
+ v -= (float) c * s.freq + s.present;
+ }
+ }
+}
+
+} // namespace strata::qwen35
diff --git a/src/qwen35/q36_engine.cu b/src/qwen35/q36_engine.cu
index 03660de40..6a773837d 100644
--- a/src/qwen35/q36_engine.cu
+++ b/src/qwen35/q36_engine.cu
@@ -26,6 +26,7 @@
#include "strata/prefill/gemm.hpp"
#include "strata/prefill/moe_mmq.hpp"
#include "strata/qwen35/engine.hpp"
+#include "strata/qwen35/sampling.hpp"
#include "q36_kernels.cuh"
#if defined(STRATA_Q36_CPU_POOL)
#include "strata/kernels/cpu/native_expert.hpp"
@@ -3009,24 +3010,10 @@ bool place_experts(Model& m, Engine& E, double vram_gib, int threads, int hybrid
}
// ------------------------------------------------------------------------------------------------ sampling
-struct Sampling {
- float temperature = 0.f, top_p = 1.f, min_p = 0.f, repeat = 1.f, freq = 0.f, present = 0.f;
- int top_k = 20, last_n = 0;
- unsigned long long seed = 0;
- bool needs_logits() const { return temperature > 0.f || repeat != 1.f || freq != 0.f || present != 0.f; }
-};
+using strata::qwen35::Sampling;
int sample(std::vector<float>& lg, const Sampling& s, const std::vector<int32_t>& hist, std::mt19937_64& rng) {
- if (s.last_n != 0 && (s.repeat != 1.f || s.freq != 0.f || s.present != 0.f)) {
- std::map<int, int> cnt;
- const size_t from = s.last_n < 0 || (size_t) s.last_n >= hist.size() ? 0 : hist.size() - (size_t) s.last_n;
- for (size_t i = from; i < hist.size(); ++i) ++cnt[hist[i]];
- for (const auto& [id, c] : cnt) {
- float& v = lg[(size_t) id];
- if (s.repeat != 1.f) v = v > 0 ? v / s.repeat : v * s.repeat;
- v -= (float) c * s.freq + s.present;
- }
- }
+ strata::qwen35::apply_penalties(lg, s, hist);
if (s.temperature <= 0.f) return (int) (std::max_element(lg.begin(), lg.end()) - lg.begin());
const int kk = std::max(1, std::min(s.top_k > 0 ? s.top_k : 64, (int) lg.size()));
std::vector<int> idx(lg.size());
diff --git a/src/qwen35/sampling_test.cpp b/src/qwen35/sampling_test.cpp
new file mode 100644
index 000000000..4b8e2a590
--- /dev/null
+++ b/src/qwen35/sampling_test.cpp
@@ -0,0 +1,112 @@
+// The qwen35 host sampler's penalties (include/strata/qwen35/sampling.hpp), CPU only, no model.
+// #3911: a history token outside a short logit row (--draft-vocab N) must not write past the row.
+#include "strata/qwen35/sampling.hpp"
+
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <map>
+
+using strata::qwen35::Sampling;
+using strata::qwen35::apply_penalties;
+
+namespace {
+int checks = 0;
+void check(bool ok, const char* label) {
+ ++checks;
+ if (!ok) { std::fprintf(stderr, "FAIL: %s\n", label); std::exit(1); }
+}
+
+// The loop as it was before the guard (origin/main of the time), for the in-range comparison.
+void old_penalties(std::vector<float>& lg, const Sampling& s, const std::vector<int32_t>& hist) {
+ if (s.last_n != 0 && (s.repeat != 1.f || s.freq != 0.f || s.present != 0.f)) {
+ std::map<int, int> cnt;
+ const size_t from = s.last_n < 0 || (size_t) s.last_n >= hist.size() ? 0 : hist.size() - (size_t) s.last_n;
+ for (size_t i = from; i < hist.size(); ++i) ++cnt[hist[i]];
+ for (const auto& [id, c] : cnt) {
+ float& v = lg[(size_t) id];
+ if (s.repeat != 1.f) v = v > 0 ? v / s.repeat : v * s.repeat;
+ v -= (float) c * s.freq + s.present;
+ }
+ }
+}
+
+bool same_bits(const std::vector<float>& a, const std::vector<float>& b) {
+ return a.size() == b.size() && std::memcmp(a.data(), b.data(), a.size() * sizeof(float)) == 0;
+}
+} // namespace
+
+int main() {
+ Sampling s;
+ s.repeat = 1.2f; s.freq = 0.1f; s.present = 0.05f; s.last_n = -1;
+
+ // a short row (a 64-token draft vocabulary) and a history with tokens inside it, past it and negative. The row's
+ // storage past its size is filled with a marker first: a write past the row would change the marker.
+ {
+ std::vector<float> row(256, 7.f);
+ row.resize(64);
+ for (int i = 0; i < 64; ++i) row[(size_t) i] = (float) i - 20.f;
+ const std::vector<float> before(row.begin(), row.end());
+ const std::vector<int32_t> hist = {3, 100, 3, 63, 64, 65, 255, 100000, -1, 7};
+ apply_penalties(row, s, hist);
+ for (int i = 64; i < 256; ++i) check(row.data()[i] == 7.f, "nothing is written past the row");
+ check(row.size() == 64, "the row keeps its size");
+ // tokens 3 (twice), 63 and 7 are in the row and are penalised; the others are skipped
+ std::vector<float> want = before;
+ for (int id : {3, 7, 63}) {
+ const int c = id == 3 ? 2 : 1;
+ float& v = want[(size_t) id];
+ v = v > 0 ? v / s.repeat : v * s.repeat;
+ v -= (float) c * s.freq + s.present;
+ }
+ check(same_bits(row, want), "the in-row tokens take the penalty, the rest of the row is untouched");
+ }
+
+ // history tokens that are all outside the row: the row does not change
+ {
+ std::vector<float> row(16, 1.5f);
+ const std::vector<float> before = row;
+ apply_penalties(row, s, {16, 17, 1000, -5});
+ check(same_bits(row, before), "only out-of-row tokens: no change");
+ }
+
+ // a row of the whole vocabulary behaves exactly as before the guard, bit for bit, for every window and penalty mix
+ {
+ const int V = 512;
+ std::vector<float> base((size_t) V);
+ for (int i = 0; i < V; ++i) base[(size_t) i] = ((float) ((i * 37) % 101) - 50.f) / 7.f;
+ std::vector<int32_t> hist;
+ for (int i = 0; i < 300; ++i) hist.push_back((i * 53 + 11) % V);
+ const float repeats[] = {1.f, 1.1f, 1.5f};
+ const float freqs[] = {0.f, 0.2f};
+ const float presents[] = {0.f, 0.3f};
+ const int lasts[] = {0, -1, 1, 64, 300, 5000};
+ for (float r : repeats) for (float f : freqs) for (float p : presents) for (int n : lasts) {
+ Sampling t;
+ t.repeat = r; t.freq = f; t.present = p; t.last_n = n;
+ std::vector<float> a = base, b = base;
+ apply_penalties(a, t, hist);
+ old_penalties(b, t, hist);
+ check(same_bits(a, b), "full-vocabulary row: identical to the unguarded loop");
+ }
+ }
+
+ // no penalty configured: untouched even with a history
+ {
+ Sampling t;
+ std::vector<float> row(8, 2.f);
+ const std::vector<float> before = row;
+ apply_penalties(row, t, {1, 2, 99});
+ check(same_bits(row, before), "no penalty: no change");
+ }
+
+ // an empty row with a history does not crash
+ {
+ std::vector<float> row;
+ apply_penalties(row, s, {0, 1, 2});
+ check(row.empty(), "empty row");
+ }
+
+ std::printf("qwen35_sampling_test: %d checks passed\n", checks);
+ return 0;
+}