Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Merge f-027-gh-eng into f-027-gh-eng-pm (#4121 on top of the #4010 pipelined adaptive rounds)

Niko1221bd8a25445388
diff --cc src/program/batch_groups_test.cpp
index 564ad3dc9,037defe6f..3c9788b4a
--- a/src/program/batch_groups_test.cpp
+++ b/src/program/batch_groups_test.cpp
@@@ -3,9 -3,10 +3,11 @@@
// 1. `auto` picks the most groups, at most one per stage, that divide the slots;
// 2. a layer split with --batch >= 2 and no --batch-groups means `auto` (0.1.41);
// 3. --batch-mtp (or --adapt-async 1) turns that default into one group, and says which feature did it;
+// with STRATA_PIPED_MTP and one slot per group, --batch-mtp keeps the pipeline and drafts in it;
// 4. a number or `auto` given explicitly is honoured, with or without those features;
- // 5. no layer split: the number given (1) stays.
+ // 5. no layer split: the number given (1) stays;
+ // 6. STRATA_PIPE_ADAPT_EVERY: off unless a positive number; a round is due after N group windows, never with the tier
+ // off, beside --adapt-async, while a prompt is read or while the last round's copies are in flight.
#include "strata/program/batch_groups.hpp"
#include <cstdio>
diff --cc src/program/generate.cpp
index cb36cd56a,705931fb7..be5522caa
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@@ -12032,159 -12656,28 +12733,181 @@@ int main(int argc, char** argv)
std::mutex bell_mu;
std::condition_variable bell_cv;
std::atomic<bool> bell_waiting{false};
+ // STRATA_PIPED_MTP: a slot whose next proposals are drafting (its chain in flight on the drafter's stream), and
+ // per slot the event after the copy of its rows on the last stage's stream, which the chain waits for
+ std::vector<char> pm_drafting(bs.size(), 0);
+ std::vector<cudaEvent_t> pm_copied(pmtp ? bs.size() : 0, nullptr);
+ if (pmtp) {
+ const strata::core::OnDevice on_last(stage_verifier(n_pipe - 1).device());
+ for (cudaEvent_t& e : pm_copied)
+ if (cudaEventCreateWithFlags(&e, cudaEventDisableTiming) != cudaSuccess) {
+ std::fprintf(stderr, "strata serve: STRATA_PIPED_MTP: event creation failed\n");
+ return 1;
+ }
+ }
+ // the chain of slot b has landed: its proposals are the slot's next drafts (1), still drafting (0), or an error (-1)
+ auto pmtp_poll_draft = [&](int b, bool wait) -> int {
+ if (!pm_drafting[(size_t) b]) return 1;
+ for (;;) {
+ const int r = slot_mtp[(size_t) b]->chain_poll(err);
+ if (r < 0) { std::printf("ERR batch MTP slot %d: %s\n", b, err.c_str()); return -1; }
+ if (r == 1) break;
+ if (!wait) return 0;
+ }
+ pm_drafting[(size_t) b] = 0;
+ // the first proposal always; another while the draft layer was at least --spec-min-p sure of the one before
+ // (the solo path's rule, MtpDrafter::draft)
+ BSlot& sl = bs[(size_t) b];
+ const int32_t* tok = slot_mtp[(size_t) b]->chain_tok();
+ const float* prob = slot_mtp[(size_t) b]->chain_prob();
+ sl.n_drafts = 1;
+ sl.draft[0] = tok[0];
+ while (sl.n_drafts < pm_drafts && prob[sl.n_drafts - 1] >= (float) o.spec_min_p) {
+ sl.draft[(size_t) sl.n_drafts] = tok[sl.n_drafts];
+ ++sl.n_drafts;
+ }
+ sl.draft_ready = true;
+ return 1;
+ };
+ // STRATA_PIPED_MTP: the last stage has picked for group gi's window (its slot's token and G.S - 1 proposals).
+ // Judge the proposals, queue every stage's commit of the kept rows (each stage runs it before that group's next
+ // window there: the same stream), emit them, and start drafting the next proposals.
+ auto pmtp_verdict = [&](PGroup& G, strata::core::Verifier& vk, const int32_t* outb) -> bool {
+ auto is_eos = [&](int32_t y) {
+ return std::find(o.eos_ids.begin(), o.eos_ids.end(), (int64_t) y) != o.eos_ids.end();
+ };
+ // each slot of the group owns a contiguous block of the window's rows: its token, then its own proposals
+ struct Blk { int b = 0, f = 0, n = 0, keep = 1; };
+ std::vector<Blk> blks;
+ std::vector<int> keep(bs.size(), 1);
+ for (int t = 0; t < G.S;) {
+ const int b = G.rows[t];
+ const int f = t;
+ while (t < G.S && G.rows[t] == b) ++t;
+ BSlot& sl = bs[(size_t) b];
+ // proposal j stands when the target picked it after the row before it (and every earlier one stood); the
+ // group was started only with proposals there is room to emit
+ Blk blk;
+ blk.b = b;
+ blk.f = f;
+ blk.n = t - f;
+ while (blk.keep < blk.n && !sl.stop && outb[f + blk.keep - 1] == G.tok[f + blk.keep] &&
+ !is_eos(outb[f + blk.keep - 1]))
+ ++blk.keep;
+ keep[(size_t) b] = blk.keep;
+ bt_accepted += blk.keep - 1;
+ bt_tokens += blk.n - 1; // (proposals checked)
+ blks.push_back(blk);
+ }
+ for (int k = 0; k < n_pipe; ++k)
+ if (!stage_verifier(k).batch_commit_rows(G.rows, G.S, G.hbase, keep.data(), G.tok, G.pos, err)) {
+ std::printf("ERR %s\n", err.c_str());
+ return false;
+ }
+ // #879: a row with non-finite logits: the slot that owns it ends, nothing of it is emitted
+ for (int t = 0; t < G.S; ++t)
+ if (vk.nonfinite_in(t)) {
+ const int b = G.rows[t];
+ BSlot& sl = bs[(size_t) b];
+ std::fprintf(stderr, "strata batch: non-finite logits (#879) in slot %d at position %lld: it ends (BDONE "
+ "nonfinite) and the next request reads its prompt from token 0\n", b, (long long) sl.p);
+ const double ms = std::chrono::duration<double, std::milli>(Clock::now() - sl.t0).count();
+ std::printf("BDONE %d %lld nonfinite %.1f\n", b, (long long) sl.produced, ms);
+ sl.active = false;
+ sl.cached = false;
+ sl.draft_ready = false;
+ sl.checks.clear();
+ nan_flush = true;
+ }
+ for (const Blk& blk : blks) {
+ BSlot& sl = bs[(size_t) blk.b];
+ if (!sl.active) continue; // it ended above (non-finite logits) or before this window
+ for (int j = 0; j < blk.keep; ++j) {
+ const int32_t y = outb[blk.f + j];
+ std::printf("BT %d %d\n", blk.b, (int) y);
+ ++sl.produced;
+ ++bt_rows;
+ ++pipe_rows;
+ const char* fin = is_eos(y) ? "stop" : sl.stop ? "cancel" : sl.produced >= sl.max_new ? "length"
+ : sl.p + 2 > o.max_context ? "length" : nullptr;
+ sl.ids.push_back(G.tok[blk.f + j]);
+ if (fin != nullptr) {
+ const double ms = std::chrono::duration<double, std::milli>(Clock::now() - sl.t0).count();
+ std::printf("BDONE %d %lld %s %.1f\n", blk.b, (long long) sl.produced, fin, ms);
+ sl.active = false;
+ sl.draft_ready = false;
+ // a cache again (#857), as on the pipelined path without drafts: final once its commits have run
+ // (pmtp_settle, before anything reads the slot's sessions)
+ sl.cached = o.prompt_cache > 0 && !sl.img && !sl.tainted;
+ break;
+ }
+ sl.x = y;
+ sl.p += 1;
+ }
+ }
+ // the next proposal: each slot's drafter catches up on ITS OWN rows of this window and drafts from the last
+ // kept one, as a chain on its own stream (pmtp_poll_draft collects it before the group's next window). The
+ // rows - the last stage's final residuals - are copied on THAT stage's stream, so its next window, which
+ // rewrites them, runs after the copy; the chain waits for the copy.
+ const size_t stride = (size_t) g.hc * (size_t) g.n_embd;
+ for (const Blk& blk : blks) {
+ BSlot& sl = bs[(size_t) blk.b];
+ if (!sl.active) continue; // it ended: no more drafts for it
+ const strata::core::OnDevice on_slot_mtp(slot_mtp[(size_t) blk.b]->device());
+ sl.draft_ready = false;
+ const float* src = vk.final_R_all() + (size_t) blk.f * stride;
+ if (cudaMemcpyAsync(slot_mtp_rows[(size_t) blk.b].get(), src, (size_t) blk.n * stride * sizeof(float),
+ cudaMemcpyDeviceToDevice, vk.stream()) != cudaSuccess ||
+ cudaEventRecord(pm_copied[(size_t) blk.b], vk.stream()) != cudaSuccess ||
+ cudaStreamWaitEvent(slot_mtp[(size_t) blk.b]->stream(), pm_copied[(size_t) blk.b], 0) != cudaSuccess ||
+ !slot_mtp[(size_t) blk.b]->chain_launch(blk.n, outb + blk.f, G.pos[blk.f], blk.keep - 1, nullptr, 0,
+ pm_drafts, 0, err)) {
+ std::printf("ERR batch MTP slot %d: %s\n", blk.b,
+ err.empty() ? "the proposal's chain did not start" : err.c_str());
+ return false;
+ }
+ pm_drafting[(size_t) blk.b] = 1;
+ }
+ return true;
+ };
+ // STRATA_PIPED_MTP: the commits queued after the verdicts have run on every stage, so whatever reads a slot's
+ // sessions next (an admission, a parked conversation, the solo path taking a slot back) sees its final state
+ auto pmtp_settle = [&]() -> bool {
+ if (!pmtp) return true;
+ for (int b = 0; b < (int) bs.size(); ++b) // (a chain in flight would refuse the drafter's next use)
+ if (pmtp_poll_draft(b, true) < 0) return false;
+ for (int k = 0; k < n_pipe; ++k) {
+ strata::core::Verifier& vk = stage_verifier(k);
+ const strata::core::OnDevice on_k(vk.device());
+ if (cudaStreamSynchronize(vk.stream()) != cudaSuccess) {
+ std::printf("ERR a pipelined commit failed on stage %d\n", k + 1);
+ return false;
+ }
+ }
+ return true;
+ };
+ // STRATA_PIPE_ADAPT_EVERY=N (opt-in; unset or 0: off, as before): the pipelined groups start no adaptive-tier round
+ // of their own (batch_step starts one every --adapt-every windows; pump below never does), so under steady parallel
+ // load the VRAM tier only moves when the slots go back to the solo path, and a fresh engine keeps its start-up
+ // profile for minutes. With N > 0, after N group windows the pipeline starts no new group, lets the ones in flight
+ // finish, and runs one round there as batch_step runs it between serial windows (nothing in flight reads a cache
+ // slot; the swapped-in experts land before the next launch), then goes on. Never while a prompt is read beside
+ // the slots (its loan of cache slots), nor with --adapt-async (its own ticks).
+ // (the rule: batch_groups.hpp pipe_adapt_every / pipe_adapt_due, tested in batch_groups_test)
+ const int pipe_adapt_every = strata::program::batch_groups::pipe_adapt_every(std::getenv("STRATA_PIPE_ADAPT_EVERY"));
+ int64_t pa_windows = 0, pa_last = 0, pa_rounds = 0, pa_swaps = 0;
+ double pa_ms = 0;
+ auto pa_due = [&] {
+ strata::program::batch_groups::PipeAdapt p;
+ p.every = pipe_adapt_every;
+ p.tier_on = !drive.d.usage.empty();
+ p.async_tier = (bool) ajob;
+ p.prompt_read = in_prompt_read;
+ p.swaps_pending = !pending.empty();
+ p.windows = pa_windows;
+ p.last = pa_last;
+ return strata::program::batch_groups::pipe_adapt_due(p);
+ };
// one turn: serve every running stage, hand finished groups on, start what can start
auto pump = [&](bool may_start) -> bool {
++pipe_tick;
@@@ -12265,20 -12758,10 +12989,20 @@@
pick = gi;
if (pick < 0) continue;
} else {
- if (!may_start) continue;
+ if (!may_start || pa_due()) continue; // STRATA_PIPE_ADAPT_EVERY: a round is due, the pipeline empties first
for (int j = 0; j < (int) pg.size() && pick < 0; ++j) {
const int gi = (int) ((rr + j) % (int64_t) pg.size());
- if (!pg[(size_t) gi].inflight && group_active(gi)) pick = gi;
+ if (pg[(size_t) gi].inflight || !group_active(gi)) continue;
+ if (pmtp) { // its slots' proposal chains still running: another group goes first
+ bool ready = true;
+ for (int t = 0; t < GS; ++t) {
+ const int dr = pmtp_poll_draft(gi * GS + t, false);
+ if (dr < 0) return false;
+ if (dr == 0) { ready = false; break; }
+ }
+ if (!ready) continue;
+ }
+ pick = gi;
}
if (pick < 0) continue;
rr = pick + 1;
@@@ -12351,6 -12807,18 +13075,19 @@@
}
}
if (drive.d.failed) { std::printf("ERR %s\n", drive.d.fail ? drive.d.fail : "the expert pool failed"); return false; }
+ if (pa_due() && !pipe_inflight()) { // STRATA_PIPE_ADAPT_EVERY: the pipeline is empty - one round, as batch_step
++ if (pmtp && !pmtp_settle()) return false; // STRATA_PIPED_MTP: the queued commits and draft chains are done first
+ const Clock::time_point ta = Clock::now();
+ pa_last = pa_windows;
+ if (!adapt()) {
+ std::printf("ERR an adaptive refill failed\n");
+ return false;
+ }
+ ++pa_rounds;
+ pa_swaps += (int64_t) pending.size();
+ apply_pending(true); // the copies land and the tables follow (the next launch plans from them)
+ pa_ms += std::chrono::duration<double, std::milli>(Clock::now() - ta).count();
+ }
if (batch_telemetry_enabled) {
const auto sample = stage_in_flight.sample(Clock::now(), !pipe_inflight() && !batch_on());
if (sample) {
@@@ -12360,16 -12828,17 +13097,22 @@@
std::fflush(stdout);
}
}
+ // every slot done: the last commits have run before anything reads a slot (between windows nothing does)
+ if (pmtp && !pipe_inflight() && !batch_on() && !pmtp_settle()) return false;
if (!pipe_inflight() && !batch_on() && bt_windows > 0) { // all idle: the timing line, as batch_step
const double w = (double) bt_windows, wall = std::chrono::duration<double, std::milli>(Clock::now() - bt_start).count();
+ char pa_txt[160] = "";
+ if (pipe_adapt_every > 0)
+ std::snprintf(pa_txt, sizeof pa_txt, "; adaptive tier (STRATA_PIPE_ADAPT_EVERY): %lld rounds, %lld experts "
+ "swapped in, %.0f ms", (long long) pa_rounds, (long long) pa_swaps, pa_ms);
std::fprintf(stderr, "strata batch (pipelined, %d groups of %d): %lld group-steps, %lld rows in %.0f ms = "
- "%.1f rows/s (admissions included)", o.batch_groups, GS, (long long) bt_windows,
- (long long) bt_rows, wall, 1000.0 * bt_rows / std::max(wall, 1e-9));
- "%.1f rows/s (admissions included)%s\n", o.batch_groups, GS, (long long) bt_windows,
++ "%.1f rows/s (admissions included)%s", o.batch_groups, GS, (long long) bt_windows,
+ (long long) bt_rows, wall, 1000.0 * bt_rows / std::max(wall, 1e-9), pa_txt);
+ if (pmtp)
- std::fprintf(stderr, ", %lld of %lld MTP proposals kept", (long long) bt_accepted, (long long) bt_tokens);
++ std::fprintf(stderr, " (%lld of %lld MTP proposals kept)", (long long) bt_accepted, (long long) bt_tokens);
+ std::fprintf(stderr, "\n");
+ pa_rounds = pa_swaps = 0;
+ pa_ms = 0;
for (int k = 0; k < n_pipe; ++k) {
const std::string pr = stage_verifier(k).profile_report();
if (!pr.empty()) std::fprintf(stderr, "strata batch GPU stages, stage %d (ms/window):%s\n", k + 1, pr.c_str());