Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41: a long prompt skips layers 20-39 on all but its last 2,668 rows (STRATA_DS4_PROMPT_SKIP, on by default)

Niko12218a7102ec4da8
diff --git a/src/dsv41/ds4_main.cpp b/src/dsv41/ds4_main.cpp
index e614e6cb1..fb4c1824c 100644
--- a/src/dsv41/ds4_main.cpp
+++ b/src/dsv41/ds4_main.cpp
@@ -583,6 +583,16 @@ int score(strata::ds4::Model& m, const std::string& ids_path, const std::string&
std::fprintf(stderr, "strata-dsv41: %s\n", err.c_str());
return 1;
}
+ // DS4_SCORE_PREFIX=N (a test of the long-prompt layer skip): the segments are joined into one stream, its first N
+ // tokens are read as one prompt (logits of the last row only: the path a long prompt takes, where the skip applies)
+ // and only the positions after them are scored, in windows
+ const long long prefix_n = std::getenv("DS4_SCORE_PREFIX") ? std::atoll(std::getenv("DS4_SCORE_PREFIX")) : 0;
+ if (prefix_n > 0 && segs.size() > 1) {
+ ScoreSeg all;
+ all.name = "concat";
+ for (const ScoreSeg& sg : segs) all.ids.insert(all.ids.end(), sg.ids.begin(), sg.ids.end());
+ segs.assign(1, all);
+ }
std::FILE* out = std::fopen(out_path.c_str(), "wb");
std::FILE* sj = std::fopen((out_path + ".seg.json").c_str(), "wb");
if (out == nullptr || sj == nullptr) {
@@ -601,7 +611,15 @@ int score(strata::ds4::Model& m, const std::string& ids_path, const std::string&
const long long first = line;
double sum_nll = 0.0;
long long top1 = 0, cnt = 0;
- for (size_t a = 0; a + 1 < ids.size();) {
+ size_t a0 = 0;
+ if (prefix_n > 0 && (size_t) prefix_n + 1 < ids.size()) {
+ a0 = (size_t) prefix_n;
+ if (!m.feed(ids.data(), (int) a0, 1, err)) {
+ std::fprintf(stderr, "strata-dsv41: %s\n", err.c_str());
+ return 1;
+ }
+ }
+ for (size_t a = a0; a + 1 < ids.size();) {
const int T = (int) std::min<size_t>((size_t) window, ids.size() - 1 - a); // (positions a .. a+T-1 are scored)
if (!m.feed(ids.data() + a, T, T, err) || !m.logits(lg) || lg.size() < (size_t) T * (size_t) V) {
std::fprintf(stderr, "strata-dsv41: %s\n", err.empty() ? "no logits" : err.c_str());
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index a9e370441..bc29b69d8 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -583,6 +583,7 @@ struct Chunk {
int spec_k = -1, spec_il = -1, n_spec = 0; // the likely items queued last: (chunk, layer), how many, from where
size_t spec_begin = 0;
std::vector<int> fl_layers, fl_T; // the device's trunk layers in order; the run's chunk sizes
+ int fl_yoco = -1, fl_cut_k = 0, fl_cut_r = 0; // Run::yoco / cut_k / cut_r: what a layer's chunk streams for
int bc_n = 0, bc_w = 0; // a batch's compacted bounds: bc_n regions of bc_w, by batch count
int *bc_pin = nullptr, *bc_dev = nullptr;
uint64_t n_batch = 0;
@@ -813,6 +814,7 @@ struct Model::Impl {
std::vector<std::unique_ptr<Chunk>> chunks; // per device
size_t eg_cap = kWin; // tokens h_eg holds
int tap_cap = kWin; // rows h_taps holds (a chunk seeds the drafter with its last 128)
+ static constexpr int kYocoMin = 1024; // a long prompt's skip (forward_chunks): the fewest rows worth skipping
bool logits_all = false; // the last feed was a chunk with logits for every row (chunk_logits)
std::vector<float> chunk_logits;
std::function<void(int64_t)> progress; // a run of chunks: the tokens done so far (the main thread, while it waits)
@@ -3203,8 +3205,22 @@ struct Model::Impl {
if (v >= l.n_slots + c.n_spec && b[(size_t) v + 1] > b[(size_t) v]) flow_push(c, l, mi, k, e, v);
}
}
- // the next (chunk, layer) of the device after (k, il) in the run's order (false: none)
+ // the next (chunk, layer) of the device after (k, il) in the run's order that streams experts (false: none) - a
+ // long prompt's skip leaves out its early chunks' layers from Run::yoco on (kv_only there: no experts)
bool flow_next(const Chunk& c, int k, int il, int& k2, int& il2) const {
+ while (flow_next1(c, k, il, k2, il2)) {
+ if (c.fl_yoco < 0 || k2 >= c.fl_cut_k || il2 < c.fl_yoco) return true;
+ k = k2;
+ il = il2;
+ }
+ return false;
+ }
+ // the rows chunk k runs layer il on (a long prompt's skip: the cut chunk's from cut_r on, from the yoco layer on)
+ static int fl_rows(const Chunk& c, int k, int il) {
+ const int T = c.fl_T[(size_t) k];
+ return c.fl_yoco >= 0 && k == c.fl_cut_k && il >= c.fl_yoco ? T - c.fl_cut_r : T;
+ }
+ bool flow_next1(const Chunk& c, int k, int il, int& k2, int& il2) const {
const auto it = std::find(c.fl_layers.begin(), c.fl_layers.end(), il);
const size_t i = (size_t) (it - c.fl_layers.begin());
if (c.lm) { // (layer-major: the layer's next chunk, then the next layer's first)
@@ -3345,7 +3361,7 @@ struct Model::Impl {
flow_rest(c, k, L0.il, b);
const size_t lb = c.spec_begin, le = c.base + c.items.size(), n_lay = le - lb;
int k2 = 0, il2 = 0;
- if (flow_next(c, k, L0.il, k2, il2)) flow_spec(c, k2, il2, c.fl_T[(size_t) k2]);
+ if (flow_next(c, k, L0.il, k2, il2)) flow_spec(c, k2, il2, fl_rows(c, k2, il2));
const size_t R = (size_t) c.ring_n, bmax = std::max<size_t>(1, R / 2);
int v_done = 0;
while (c.n_comp < le) {
@@ -6219,8 +6235,10 @@ struct Model::Impl {
// layer L0 for a chunk of T tokens at p0 as group member mi: layer()'s arithmetic over the chunk's rows (the
// projections as GEMMs, the attention and the indexer in slabs, the routed experts as sorted rows: the VRAM ones in
- // place, the CPU lane's streamed in batches)
- bool layer_chunk(Layer& L0, int mi, int64_t p0, int T, const WinPlan& wp, Streamer& S, std::string& err) {
+ // place, the CPU lane's streamed in batches). kv_only (a long prompt's first pass, Run::stop): only what the layers
+ // after it read of these rows - its compressed KV and index keys - and nothing of its own attention or experts
+ bool layer_chunk(Layer& L0, int mi, int64_t p0, int T, const WinPlan& wp, Streamer& S, std::string& err,
+ bool kv_only = false) {
Layer& l = member(L0, mi);
Dev& d = devs[(size_t) l.dev];
Chunk& c = *chunks[(size_t) l.dev];
@@ -6252,35 +6270,37 @@ struct Model::Impl {
hc_split(c.mix, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps, T, c.a_pre, c.a_post, c.a_comb, s);
hc_pre_norm(c.h, c.pre, l.attn_norm, D, H, T, g.eps, c.x, s);
x3::rows_f16(c.x, D, nullptr, T, D, c.x16, s);
- if (!gemm(d, c, l.wq_a, c.x16, D, T, c.qr, g.q_lora, 0.0f, err) ||
- !gemm(d, c, l.wkv, c.x16, D, T, c.kvr, HD, 0.0f, err))
+ if (!kv_only && (!gemm(d, c, l.wq_a, c.x16, D, T, c.qr, g.q_lora, 0.0f, err) ||
+ !gemm(d, c, l.wkv, c.x16, D, T, c.kvr, HD, 0.0f, err)))
return false;
if (l.kv_src && !gemm(d, c, l.c_wkv, c.x16, D, T, c.ckr, HD, 0.0f, err)) return false;
if (l.kv_src && l.ratio > 1 && !gemm(d, c, l.c_wgate, c.x16, D, T, c.cgr, HD, 0.0f, err)) return false;
- Finish fq;
- fq.w = l.q_norm;
- fq.eps = g.eps;
- finish(c.qr, T, g.q_lora, g.q_lora, fq, s);
- x3::rows_f16(c.qr, g.q_lora, nullptr, T, g.q_lora, c.qr16, s);
- // the window rows: the ring's (positions p0 - win ..) then the chunk's
- ring_to_ext(l.win, c.ext, p0, g.win, HD, s);
- Finish fk;
- fk.w = l.kv_norm;
- fk.eps = g.eps;
- fk.rd = g.rd;
- fk.pos = (int) p0;
- fk.freqs = freqs;
- fk.quant = 1;
- fk.dst = c.ext + (size_t) g.win * HD;
- fk.dst_stride = HD;
- finish(c.kvr, T, HD, HD, fk, s);
- // feed_mark's checkpoint: the ring as it was before row snap_t - the old ring, then the chunk's rows before it
- const bool snap = wp.snap_t >= 0 && mi == 0 && L0.snap_win != nullptr;
- if (snap) {
- CK(cudaMemcpyAsync(L0.snap_win, l.win, (size_t) g.win * win_rb(), cudaMemcpyDeviceToDevice, s));
- if (wp.snap_t > 0) ext_to_ring(c.ext, L0.snap_win, p0, g.win, wp.snap_t, HD, s, nullptr);
- }
- ext_to_ring(c.ext, l.win, p0, g.win, T, HD, s, nullptr);
+ const bool snap = !kv_only && wp.snap_t >= 0 && mi == 0 && L0.snap_win != nullptr;
+ if (!kv_only) {
+ Finish fq;
+ fq.w = l.q_norm;
+ fq.eps = g.eps;
+ finish(c.qr, T, g.q_lora, g.q_lora, fq, s);
+ x3::rows_f16(c.qr, g.q_lora, nullptr, T, g.q_lora, c.qr16, s);
+ // the window rows: the ring's (positions p0 - win ..) then the chunk's
+ ring_to_ext(l.win, c.ext, p0, g.win, HD, s);
+ Finish fk;
+ fk.w = l.kv_norm;
+ fk.eps = g.eps;
+ fk.rd = g.rd;
+ fk.pos = (int) p0;
+ fk.freqs = freqs;
+ fk.quant = 1;
+ fk.dst = c.ext + (size_t) g.win * HD;
+ fk.dst_stride = HD;
+ finish(c.kvr, T, HD, HD, fk, s);
+ // feed_mark's checkpoint: the ring as it was before row snap_t - the old ring, then the chunk's rows before it
+ if (snap) {
+ CK(cudaMemcpyAsync(L0.snap_win, l.win, (size_t) g.win * win_rb(), cudaMemcpyDeviceToDevice, s));
+ if (wp.snap_t > 0) ext_to_ring(c.ext, L0.snap_win, p0, g.win, wp.snap_t, HD, s, nullptr);
+ }
+ ext_to_ring(c.ext, l.win, p0, g.win, T, HD, s, nullptr);
+ }
const uint8_t* ckv = nullptr;
const long long ss = (long long) o.max_context + 64;
if (l.ratio > 0) {
@@ -6328,7 +6348,7 @@ struct Model::Impl {
fi.pdst_stride = (long long) ik_rb();
finish(c.ikr, ng, g.ix_hd, g.ix_hd, fi, s);
}
- if (clen_max > 0) {
+ if (clen_max > 0 && !kv_only) {
if (!gemm_f16w(c, l.i_wproj, g.ix_heads, D, c.x16, D, T,
1.0f / std::sqrt((float) g.ix_hd) / std::sqrt((float) g.ix_heads), c.iw, 64, err))
return false;
@@ -6409,6 +6429,7 @@ struct Model::Impl {
}
ckv = sr.ckv[(size_t) l.dev];
}
+ if (kv_only) return true;
// the attention in slabs: q, the sparse attention, wo_a, wo_b (this member's share) into acc
if (NHL == 0) CK(cudaMemsetAsync(c.acc, 0, (size_t) T * D * sizeof(*c.acc), s)); // (no heads)
for (int a0 = 0; a0 < T && NHL > 0; a0 += c.A) {
@@ -6629,7 +6650,39 @@ struct Model::Impl {
std::vector<int> T;
std::vector<unsigned> seq;
int64_t snap_pos = -1; // feed_mark: the position whose state before it the layers keep (-1: none)
+ // a long prompt's skip (forward_chunks): the cut at row cut_r of chunk cut_k - the chunks before it go through
+ // the layers before `yoco`, that layer as layer_chunk's kv_only and none after it; chunk cut_k's rows before
+ // cut_r the same, its rows from cut_r on and the chunks after it through every layer (yoco -1: no skip)
+ int yoco = -1, cut_k = 0, cut_r = 0;
+ bool runs(int k, int il) const { return yoco < 0 || k >= cut_k || il <= yoco; }
+ bool kv_only(int k, int il) const { return yoco >= 0 && k < cut_k && il == yoco; }
+ int r0(int k, int il) const { return yoco >= 0 && k == cut_k && il >= yoco ? cut_r : 0; } // the first row
};
+ // layer L0 over chunk k per a long prompt's skip (Run): its kv_only, or its rows from Run::r0 on - the chunk's
+ // buffers seen from that row (the yoco layer: its kv_only over the rows before it first)
+ bool run_layer(Layer& L0, int mi, int64_t p0, int T, const WinPlan& wp, Streamer& S, std::string& err,
+ const Run& run, int k, int il) {
+ if (run.kv_only(k, il)) return layer_chunk(L0, mi, p0, T, wp, S, err, true);
+ const int r0 = run.r0(k, il);
+ if (r0 <= 0) return layer_chunk(L0, mi, p0, T, wp, S, err);
+ if (il == run.yoco) {
+ WinPlan w0 = wp;
+ w0.snap_t = -1;
+ if (!layer_chunk(L0, mi, p0, r0, w0, S, err, true)) return false;
+ }
+ Chunk& c = *chunks[(size_t) member(L0, mi).dev];
+ float* const h0 = c.h;
+ float* const pre0 = c.pre;
+ c.h += (size_t) r0 * g.hc * g.dim;
+ c.pre += (size_t) r0 * 8;
+ WinPlan w1 = wp;
+ w1.snap_t = wp.snap_t >= r0 ? wp.snap_t - r0 : -1;
+ const bool ok = layer_chunk(L0, mi, p0 + r0, T - r0, w1, S, err);
+ c.h = h0;
+ c.pre = pre0;
+ return ok;
+ }
+
// layer-major (c.lm: a run of several chunks, device di every trunk layer, the flow set up by run_dev_chunks):
// layer after layer, each through all the chunks in order - a chunk's state to host memory after each of its
// layers and back before the next - so a layer's experts are read from the files once for the whole run (the
@@ -6647,7 +6700,8 @@ struct Model::Impl {
std::vector<std::pair<int, int>> egu; // (Engram layer, chunk)
for (int il : c.fl_layers)
if (L[(size_t) il].eg >= 0)
- for (int k = 0; k < nk; ++k) egu.push_back({L[(size_t) il].eg, k});
+ for (int k = 0; k < nk; ++k)
+ if (run.runs(k, il)) egu.push_back({L[(size_t) il].eg, k});
const size_t eg_per = (size_t) (pk.eg_ngram - 1) * pk.eg_heads * (pk.eg_hd + pk.eg_hd / 32);
auto read_eg = [this, &c, &run, &egu, eg_per](size_t u) {
if (u >= 2) cudaEventSynchronize(c.eg_ev[u % 2]);
@@ -6662,6 +6716,7 @@ struct Model::Impl {
Layer& L0 = L[(size_t) il];
const bool last_l = li + 1 == c.fl_layers.size();
for (int k = 0; k < nk; ++k) {
+ if (!run.runs(k, il)) continue; // (a skipped early chunk: nothing after the yoco layer)
const int64_t p0 = run.p0[(size_t) k];
const int T = run.T[(size_t) k], slot = k % 2;
const bool last_k = k + 1 == nk;
@@ -6703,8 +6758,10 @@ struct Model::Impl {
(size_t) nr, cudaMemcpyDeviceToHost, s));
}
}
- if (!layer_chunk(L0, 0, p0, T, wp, S, err)) return false;
+ const bool kvo = run.kv_only(k, il);
+ if (!run_layer(L0, 0, p0, T, wp, S, err, run, k, il)) return false;
if (dump_l != nullptr) dump_layer(d, c.h, il, p0, T);
+ if (kvo) continue; // (no layer after it reads this chunk's state)
if (!last_l) { // the chunk's state kept for the next layer
const Xfer x = chunk_xfer(c.lm_state + (size_t) k * hb, c, T, true);
for (int i = 0; i < x.n; ++i)
@@ -6756,6 +6813,9 @@ struct Model::Impl {
for (int m = 0; m < L[(size_t) il].nm; ++m)
if (member(L[(size_t) il], m).dev == di) c.fl_layers.push_back(il);
c.fl_T = run.T;
+ c.fl_yoco = run.yoco;
+ c.fl_cut_k = run.cut_k;
+ c.fl_cut_r = run.cut_r;
c.spec_k = c.spec_il = -1;
c.wait_ms = 0;
c.it_file = c.it_ram = c.it_spec = c.it_spec_unused = c.it_lc = 0;
@@ -6772,6 +6832,7 @@ struct Model::Impl {
}
for (int k = 0; k < nk && !c.flow; ++k)
for (int il = 0; il < g.nl; ++il) {
+ if (!run.runs(k, il) || run.kv_only(k, il)) continue; // (a long prompt's skip: no experts there)
const Layer& L0 = L[(size_t) il];
bool here = false;
for (int m = 0; m < L0.nm; ++m) here = here || member(L0, m).dev == di;
@@ -6827,7 +6888,7 @@ struct Model::Impl {
wp.snap_t = run.snap_pos >= p0 && run.snap_pos < p0 + T ? (int) (run.snap_pos - p0) : -1;
bool started = false;
std::vector<char> got(srcs.size(), 0);
- for (int il = 0; il < g.nl; ++il) {
+ for (int il = 0; il < g.nl && run.runs(k, il); ++il) {
Layer& L0 = L[(size_t) il];
int mi = -1;
for (int m = 0; m < L0.nm; ++m)
@@ -6882,7 +6943,7 @@ struct Model::Impl {
(size_t) nr, cudaMemcpyDeviceToHost, s));
}
}
- if (!layer_chunk(L0, mi, p0, T, wp, S, err)) return false;
+ if (!run_layer(L0, mi, p0, T, wp, S, err, run, k, il)) return false;
if (dump_l != nullptr && mi == 0) dump_layer(d, c.h, il, p0, T);
const bool last = il + 1 == g.nl || L[(size_t) il + 1].grp != L0.grp;
if (last && mi == 0 && il + 1 < g.nl) { // the hand-off, once the next group took the chunk before last
@@ -7267,6 +7328,33 @@ struct Model::Impl {
}
}
+ // the first layer of a long prompt's second half (forward_chunks' two passes), or -1: the first with compress ratio 1
+ // (V4.1: 20), every one after it ratio 1 too and owning no compressed KV or index keys of its own (they read its),
+ // every trunk layer on one GPU. The settings' MAYA_DS4_PROMPT_SKIP (STRATA_DS4_PROMPT_SKIP; DS4_YOCO for tests):
+ // 1 (the default) the skip, 2 DeepSeek's own serving's (the last 128 rows only: faster, the answers shift a
+ // little), 0 off
+ static int yoco_mode() {
+ static const int m = [] {
+ const char* v = std::getenv("DS4_YOCO");
+ if (v == nullptr) v = std::getenv("STRATA_DS4_PROMPT_SKIP");
+ return v != nullptr ? std::atoi(v) : 1;
+ }();
+ return m;
+ }
+ int yoco_layer() const {
+ if (yoco_mode() == 0 || chunk <= 0 || g.nl < 2 || (int) g.ratios.size() < g.nl) return -1;
+ int yl = -1;
+ for (int il = 0; il < g.nl && yl < 0; ++il)
+ if (g.ratios[(size_t) il] == 1) yl = il;
+ if (yl <= 0 || !L[(size_t) yl].kv_src) return -1;
+ for (int il = 0; il < g.nl; ++il) {
+ const Layer& l = L[(size_t) il];
+ if (l.nm != 1 || l.dev != L[0].dev) return -1;
+ if (il > yl && (g.ratios[(size_t) il] != 1 || l.kv_src)) return -1;
+ }
+ return yl;
+ }
+
// n tokens at positions pos .. in chunks of equal size (at most `chunk`), pipelined over the groups; the logits of
// the last min(logits_rows, 16) rows (logits_rows > 16: a run of one chunk, every row's into chunk_logits)
bool forward_chunks(const int* tk, int n, int logits_rows, std::string& err, int snap_slot = -1) {
@@ -7280,13 +7368,34 @@ struct Model::Impl {
err = "token " + std::to_string(tk[t]) + " out of range";
return false;
}
+ // a long prompt's skip: the layers from yoco_layer() on (V4.1: 20-39) read the earlier positions only through
+ // that layer's compressed KV and index keys, and each looks back one window (win - 1 positions) more than the
+ // layer before - so they need their own work only on the last keep = (nl - yoco) (win - 1) + win rows for the
+ // run's last rows and the rings the decode reads to be what the full work gives (up to the experts' summing
+ // order). The chunks stay as they are: the ones before the cut go through the layers before yoco and its
+ // kv_only, the cut's chunk through those over all its rows and through the rest from the cut on (Run::r0, a
+ // view of its buffers), the chunks after it through every layer - each layer's experts read as without the
+ // skip. One GPU (DS4_YOCO=1; 0: off); not with every row's logits (tests)
+ const int yl = logits_rows > kWin ? -1 : yoco_layer();
+ // (DS4_YOCO_KEEP=<rows>: fewer - DeepSeek's own serving replays the last 128 tokens through those layers
+ // (DeepInfer's "SWA bounded replay"), which shifts the answers a little; at least one window)
+ static const int keep_env = std::getenv("DS4_YOCO_KEEP") ? std::atoi(std::getenv("DS4_YOCO_KEEP")) : 0;
+ const int keep = yl <= 0 ? 0
+ : keep_env > 0 ? std::max(keep_env, g.win)
+ : yoco_mode() == 2 ? g.win : (g.nl - yl) * (g.win - 1) + g.win;
+ const int a = yl > 0 && n - keep >= kYocoMin ? n - keep : 0;
const int nc = (n + chunk - 1) / chunk, per = (n + nc - 1) / nc;
Run run;
- for (int a = 0; a < n; a += per) {
- run.p0.push_back(pos + a);
- run.T.push_back(std::min(per, n - a));
+ for (int x = 0; x < n; x += per) {
+ run.p0.push_back(pos + x);
+ run.T.push_back(std::min(per, n - x));
run.seq.push_back(++win_seq);
}
+ if (a > 0) {
+ run.yoco = yl;
+ run.cut_k = a / per;
+ run.cut_r = a - run.cut_k * per;
+ }
logits_all = logits_rows > kWin;
if (logits_all) chunk_logits.resize((size_t) n * g.vocab);
for (int t = 0; t < n; ++t) {
diff --git a/third_party/project-maya/NOTICE.md b/third_party/project-maya/NOTICE.md
index 0b3bf289f..40c6ddb86 100644
--- a/third_party/project-maya/NOTICE.md
+++ b/third_party/project-maya/NOTICE.md
@@ -9,3 +9,7 @@ MIT License, the full text is in `LICENSE` beside this file. The notice stays wi
Changes made while porting into Strata are listed in the git history of the `src/dsv41` directory
(the first commit of each file is the unmodified upstream copy).
+
+Later changes taken from Project Maya releases v1.1.18 to v1.1.26 (the same approach re-written onto Strata's engine,
+for example the long-prompt layer skip, the thinking budget, conversation slots) are covered by the same copyright
+and permission notice, in `LICENSE` beside this file.
diff --git a/tools/dsv41/test_prompt_skip.py b/tools/dsv41/test_prompt_skip.py
new file mode 100644
index 000000000..719c9c07e
--- /dev/null
+++ b/tools/dsv41/test_prompt_skip.py
@@ -0,0 +1,56 @@
+"""The long-prompt layer skip (STRATA_DS4_PROMPT_SKIP=1) against the full work: same answers within noise.
+
+Needs a GPU, a DeepSeek-V4.1 pack and the golden ids file, so it runs only when they are named:
+
+ STRATA_DSV41_BIN=.../strata-dsv41 STRATA_DSV41_PACK=/data/pack2.00 STRATA_DSV41_IDS=.../logpos.ids.json \
+ python -m unittest tools.dsv41.test_prompt_skip
+
+The golden segments are joined, the first DS4_SCORE_PREFIX tokens (4200) are read as one prompt (the path a long prompt
+takes: logits of the last row only, where the skip applies), and the positions after them are scored teacher-forced.
+The skip changes only what the layers 20-39 compute for the prompt's early rows, so the scored rows' KL against the
+full work must stay at the noise of the experts' summing order.
+"""
+import os
+import pathlib
+import subprocess
+import sys
+import tempfile
+import unittest
+
+sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
+import quant_quality # noqa: E402
+
+BIN = os.environ.get("STRATA_DSV41_BIN")
+PACK = os.environ.get("STRATA_DSV41_PACK")
+IDS = os.environ.get("STRATA_DSV41_IDS")
+
+
+def keep_rows(nl=40, yoco=20, win=128):
+ """The rows a long prompt keeps on layers yoco.. (the engine's forward_chunks): each layer looks back one window
+ more than the layer before, so the last (nl - yoco) * (win - 1) + win rows."""
+ return (nl - yoco) * (win - 1) + win
+
+
+class PromptSkip(unittest.TestCase):
+ def test_keep_rows_is_the_chain_of_windows(self):
+ self.assertEqual(keep_rows(), 2668)
+
+ @unittest.skipUnless(BIN and PACK and IDS, "needs STRATA_DSV41_BIN / _PACK / _IDS")
+ def test_scored_rows_after_a_skipped_prompt_match_the_full_work(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ outs = {}
+ for mode in ("0", "1"):
+ out = pathlib.Path(tmp) / f"skip{mode}.tsv"
+ env = dict(os.environ, STRATA_DS4_PROMPT_SKIP=mode, DS4_SCORE_PREFIX="4200", DS4_CHUNK="1024")
+ subprocess.run([BIN, "--pack", PACK, "--tp", "1", "--threads", "off", "--max-context", "6144", "--vision",
+ "off", "--dspark", "off", "--expert-profile", "none", "--window", "128", "--score", IDS,
+ "--score-out", str(out)], check=True, env=env)
+ outs[mode] = quant_quality.read_run(str(out))
+ res = quant_quality.compare(outs["0"], outs["1"])["all"]
+ self.assertGreater(res["positions"], 500)
+ self.assertLess(res["kl"], 0.002)
+ self.assertGreaterEqual(res["top1"], 0.99)
+
+
+if __name__ == "__main__":
+ unittest.main()