Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

SetArgs: the Metal fields go last, so the CUDA/HIP kernels' parameter layout (and SASS) is exactly v0.2.7's

Niko1221232fa357fd08
diff --git a/include/strata/dsv41/exl3.hpp b/include/strata/dsv41/exl3.hpp
index 6fb409e62..a140fceed 100644
--- a/include/strata/dsv41/exl3.hpp
+++ b/include/strata/dsv41/exl3.hpp
@@ -152,14 +152,15 @@ struct SetArgs {
const int* bounds = nullptr;
int row0 = 0;
int max_rows = 0; // the most rows one expert of the set has
- const int* hbounds = nullptr; // (Metal) the same bounds on the host: [n_exp + 1], absolute rows
- int total_rows = 0; // (Metal) the set's rows in all
int n_embd = 4096, n_ff = 2048;
float limit = 0.0f;
// gate / up: the caller's F16 copy of the activations (row_idx picks the rows). The tiles round F32 to F16 for
// their shared-memory tile anyway, so the same values come out; this only saves the F32 read of the same rows by
// every column block of an expert (18-36 reads of 1.8-3.7 MB on a prompt) - MarLin's "small operand cheap".
const uint16_t* x16 = nullptr;
+ // (Metal, last so that the CUDA / HIP layout is what it was)
+ const int* hbounds = nullptr; // the same bounds on the host: [n_exp + 1], absolute rows
+ int total_rows = 0; // the set's rows in all
};
/// h[local row] = FP16 clamped swiglu of gate and up over x[row_idx[local row]] (F32 rows, ldx floats apart).
void set_gate_up(const SetArgs& a, const float* x, int ldx, const int* row_idx, uint16_t* h16, int ldh, cudaStream_t s);
diff --git a/src/dsv41/ds4_iq.cuh b/src/dsv41/ds4_iq.cuh
index 7ff7aec67..1712ad673 100644
--- a/src/dsv41/ds4_iq.cuh
+++ b/src/dsv41/ds4_iq.cuh
@@ -94,9 +94,10 @@ struct SetArgs {
float* y = nullptr;
int ldo = 0;
int tg = 0, td = 0;
- const int* hbounds = nullptr; // (Metal) the bounds on the host: [n_exp + 1], absolute rows
- int total_rows = 0; // (Metal) the set's rows in all
int skip_le = 0; // (the tiles) the experts of at most this many rows are left out: the light kernels took them
+ // (Metal, after the CUDA / HIP fields so that their kernels' parameter layout is exactly what it was)
+ const int* hbounds = nullptr; // the bounds on the host: [n_exp + 1], absolute rows
+ int total_rows = 0; // the set's rows in all
};
bool set_gate_up(const SetArgs& a, cudaStream_t s);
bool set_down(const SetArgs& a, cudaStream_t s);