Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Merge f-027-gh-eng into f-027-gh-eng-pm (#4121 on top of the #4010 pipelined adaptive rounds)
Niko1221bd8a25445388
diff --cc src/program/batch_groups_test.cppindex 564ad3dc9,037defe6f..3c9788b4a--- a/src/program/batch_groups_test.cpp+++ b/src/program/batch_groups_test.cpp@@@ -3,9 -3,10 +3,11 @@@// 1. `auto` picks the most groups, at most one per stage, that divide the slots;// 2. a layer split with --batch >= 2 and no --batch-groups means `auto` (0.1.41);// 3. --batch-mtp (or --adapt-async 1) turns that default into one group, and says which feature did it;+// with STRATA_PIPED_MTP and one slot per group, --batch-mtp keeps the pipeline and drafts in it;// 4. a number or `auto` given explicitly is honoured, with or without those features;- // 5. no layer split: the number given (1) stays.+ // 5. no layer split: the number given (1) stays;+ // 6. STRATA_PIPE_ADAPT_EVERY: off unless a positive number; a round is due after N group windows, never with the tier+ // off, beside --adapt-async, while a prompt is read or while the last round's copies are in flight.#include "strata/program/batch_groups.hpp"#include <cstdio>diff --cc src/program/generate.cppindex cb36cd56a,705931fb7..be5522caa--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@@ -12032,159 -12656,28 +12733,181 @@@ int main(int argc, char** argv)std::mutex bell_mu;std::condition_variable bell_cv;std::atomic<bool> bell_waiting{false};+ // STRATA_PIPED_MTP: a slot whose next proposals are drafting (its chain in flight on the drafter's stream), and+ // per slot the event after the copy of its rows on the last stage's stream, which the chain waits for+ std::vector<char> pm_drafting(bs.size(), 0);+ std::vector<cudaEvent_t> pm_copied(pmtp ? bs.size() : 0, nullptr);+ if (pmtp) {+ const strata::core::OnDevice on_last(stage_verifier(n_pipe - 1).device());+ for (cudaEvent_t& e : pm_copied)+ if (cudaEventCreateWithFlags(&e, cudaEventDisableTiming) != cudaSuccess) {+ std::fprintf(stderr, "strata serve: STRATA_PIPED_MTP: event creation failed\n");+ return 1;+ }+ }+ // the chain of slot b has landed: its proposals are the slot's next drafts (1), still drafting (0), or an error (-1)+ auto pmtp_poll_draft = [&](int b, bool wait) -> int {+ if (!pm_drafting[(size_t) b]) return 1;+ for (;;) {+ const int r = slot_mtp[(size_t) b]->chain_poll(err);+ if (r < 0) { std::printf("ERR batch MTP slot %d: %s\n", b, err.c_str()); return -1; }+ if (r == 1) break;+ if (!wait) return 0;+ }+ pm_drafting[(size_t) b] = 0;+ // the first proposal always; another while the draft layer was at least --spec-min-p sure of the one before+ // (the solo path's rule, MtpDrafter::draft)+ BSlot& sl = bs[(size_t) b];+ const int32_t* tok = slot_mtp[(size_t) b]->chain_tok();+ const float* prob = slot_mtp[(size_t) b]->chain_prob();+ sl.n_drafts = 1;+ sl.draft[0] = tok[0];+ while (sl.n_drafts < pm_drafts && prob[sl.n_drafts - 1] >= (float) o.spec_min_p) {+ sl.draft[(size_t) sl.n_drafts] = tok[sl.n_drafts];+ ++sl.n_drafts;+ }+ sl.draft_ready = true;+ return 1;+ };+ // STRATA_PIPED_MTP: the last stage has picked for group gi's window (its slot's token and G.S - 1 proposals).+ // Judge the proposals, queue every stage's commit of the kept rows (each stage runs it before that group's next+ // window there: the same stream), emit them, and start drafting the next proposals.+ auto pmtp_verdict = [&](PGroup& G, strata::core::Verifier& vk, const int32_t* outb) -> bool {+ auto is_eos = [&](int32_t y) {+ return std::find(o.eos_ids.begin(), o.eos_ids.end(), (int64_t) y) != o.eos_ids.end();+ };+ // each slot of the group owns a contiguous block of the window's rows: its token, then its own proposals+ struct Blk { int b = 0, f = 0, n = 0, keep = 1; };+ std::vector<Blk> blks;+ std::vector<int> keep(bs.size(), 1);+ for (int t = 0; t < G.S;) {+ const int b = G.rows[t];+ const int f = t;+ while (t < G.S && G.rows[t] == b) ++t;+ BSlot& sl = bs[(size_t) b];+ // proposal j stands when the target picked it after the row before it (and every earlier one stood); the+ // group was started only with proposals there is room to emit+ Blk blk;+ blk.b = b;+ blk.f = f;+ blk.n = t - f;+ while (blk.keep < blk.n && !sl.stop && outb[f + blk.keep - 1] == G.tok[f + blk.keep] &&+ !is_eos(outb[f + blk.keep - 1]))+ ++blk.keep;+ keep[(size_t) b] = blk.keep;+ bt_accepted += blk.keep - 1;+ bt_tokens += blk.n - 1; // (proposals checked)+ blks.push_back(blk);+ }+ for (int k = 0; k < n_pipe; ++k)+ if (!stage_verifier(k).batch_commit_rows(G.rows, G.S, G.hbase, keep.data(), G.tok, G.pos, err)) {+ std::printf("ERR %s\n", err.c_str());+ return false;+ }+ // #879: a row with non-finite logits: the slot that owns it ends, nothing of it is emitted+ for (int t = 0; t < G.S; ++t)+ if (vk.nonfinite_in(t)) {+ const int b = G.rows[t];+ BSlot& sl = bs[(size_t) b];+ std::fprintf(stderr, "strata batch: non-finite logits (#879) in slot %d at position %lld: it ends (BDONE "+ "nonfinite) and the next request reads its prompt from token 0\n", b, (long long) sl.p);+ const double ms = std::chrono::duration<double, std::milli>(Clock::now() - sl.t0).count();+ std::printf("BDONE %d %lld nonfinite %.1f\n", b, (long long) sl.produced, ms);+ sl.active = false;+ sl.cached = false;+ sl.draft_ready = false;+ sl.checks.clear();+ nan_flush = true;+ }+ for (const Blk& blk : blks) {+ BSlot& sl = bs[(size_t) blk.b];+ if (!sl.active) continue; // it ended above (non-finite logits) or before this window+ for (int j = 0; j < blk.keep; ++j) {+ const int32_t y = outb[blk.f + j];+ std::printf("BT %d %d\n", blk.b, (int) y);+ ++sl.produced;+ ++bt_rows;+ ++pipe_rows;+ const char* fin = is_eos(y) ? "stop" : sl.stop ? "cancel" : sl.produced >= sl.max_new ? "length"+ : sl.p + 2 > o.max_context ? "length" : nullptr;+ sl.ids.push_back(G.tok[blk.f + j]);+ if (fin != nullptr) {+ const double ms = std::chrono::duration<double, std::milli>(Clock::now() - sl.t0).count();+ std::printf("BDONE %d %lld %s %.1f\n", blk.b, (long long) sl.produced, fin, ms);+ sl.active = false;+ sl.draft_ready = false;+ // a cache again (#857), as on the pipelined path without drafts: final once its commits have run+ // (pmtp_settle, before anything reads the slot's sessions)+ sl.cached = o.prompt_cache > 0 && !sl.img && !sl.tainted;+ break;+ }+ sl.x = y;+ sl.p += 1;+ }+ }+ // the next proposal: each slot's drafter catches up on ITS OWN rows of this window and drafts from the last+ // kept one, as a chain on its own stream (pmtp_poll_draft collects it before the group's next window). The+ // rows - the last stage's final residuals - are copied on THAT stage's stream, so its next window, which+ // rewrites them, runs after the copy; the chain waits for the copy.+ const size_t stride = (size_t) g.hc * (size_t) g.n_embd;+ for (const Blk& blk : blks) {+ BSlot& sl = bs[(size_t) blk.b];+ if (!sl.active) continue; // it ended: no more drafts for it+ const strata::core::OnDevice on_slot_mtp(slot_mtp[(size_t) blk.b]->device());+ sl.draft_ready = false;+ const float* src = vk.final_R_all() + (size_t) blk.f * stride;+ if (cudaMemcpyAsync(slot_mtp_rows[(size_t) blk.b].get(), src, (size_t) blk.n * stride * sizeof(float),+ cudaMemcpyDeviceToDevice, vk.stream()) != cudaSuccess ||+ cudaEventRecord(pm_copied[(size_t) blk.b], vk.stream()) != cudaSuccess ||+ cudaStreamWaitEvent(slot_mtp[(size_t) blk.b]->stream(), pm_copied[(size_t) blk.b], 0) != cudaSuccess ||+ !slot_mtp[(size_t) blk.b]->chain_launch(blk.n, outb + blk.f, G.pos[blk.f], blk.keep - 1, nullptr, 0,+ pm_drafts, 0, err)) {+ std::printf("ERR batch MTP slot %d: %s\n", blk.b,+ err.empty() ? "the proposal's chain did not start" : err.c_str());+ return false;+ }+ pm_drafting[(size_t) blk.b] = 1;+ }+ return true;+ };+ // STRATA_PIPED_MTP: the commits queued after the verdicts have run on every stage, so whatever reads a slot's+ // sessions next (an admission, a parked conversation, the solo path taking a slot back) sees its final state+ auto pmtp_settle = [&]() -> bool {+ if (!pmtp) return true;+ for (int b = 0; b < (int) bs.size(); ++b) // (a chain in flight would refuse the drafter's next use)+ if (pmtp_poll_draft(b, true) < 0) return false;+ for (int k = 0; k < n_pipe; ++k) {+ strata::core::Verifier& vk = stage_verifier(k);+ const strata::core::OnDevice on_k(vk.device());+ if (cudaStreamSynchronize(vk.stream()) != cudaSuccess) {+ std::printf("ERR a pipelined commit failed on stage %d\n", k + 1);+ return false;+ }+ }+ return true;+ };+ // STRATA_PIPE_ADAPT_EVERY=N (opt-in; unset or 0: off, as before): the pipelined groups start no adaptive-tier round+ // of their own (batch_step starts one every --adapt-every windows; pump below never does), so under steady parallel+ // load the VRAM tier only moves when the slots go back to the solo path, and a fresh engine keeps its start-up+ // profile for minutes. With N > 0, after N group windows the pipeline starts no new group, lets the ones in flight+ // finish, and runs one round there as batch_step runs it between serial windows (nothing in flight reads a cache+ // slot; the swapped-in experts land before the next launch), then goes on. Never while a prompt is read beside+ // the slots (its loan of cache slots), nor with --adapt-async (its own ticks).+ // (the rule: batch_groups.hpp pipe_adapt_every / pipe_adapt_due, tested in batch_groups_test)+ const int pipe_adapt_every = strata::program::batch_groups::pipe_adapt_every(std::getenv("STRATA_PIPE_ADAPT_EVERY"));+ int64_t pa_windows = 0, pa_last = 0, pa_rounds = 0, pa_swaps = 0;+ double pa_ms = 0;+ auto pa_due = [&] {+ strata::program::batch_groups::PipeAdapt p;+ p.every = pipe_adapt_every;+ p.tier_on = !drive.d.usage.empty();+ p.async_tier = (bool) ajob;+ p.prompt_read = in_prompt_read;+ p.swaps_pending = !pending.empty();+ p.windows = pa_windows;+ p.last = pa_last;+ return strata::program::batch_groups::pipe_adapt_due(p);+ };// one turn: serve every running stage, hand finished groups on, start what can startauto pump = [&](bool may_start) -> bool {++pipe_tick;@@@ -12265,20 -12758,10 +12989,20 @@@pick = gi;if (pick < 0) continue;} else {- if (!may_start) continue;+ if (!may_start || pa_due()) continue; // STRATA_PIPE_ADAPT_EVERY: a round is due, the pipeline empties firstfor (int j = 0; j < (int) pg.size() && pick < 0; ++j) {const int gi = (int) ((rr + j) % (int64_t) pg.size());- if (!pg[(size_t) gi].inflight && group_active(gi)) pick = gi;+ if (pg[(size_t) gi].inflight || !group_active(gi)) continue;+ if (pmtp) { // its slots' proposal chains still running: another group goes first+ bool ready = true;+ for (int t = 0; t < GS; ++t) {+ const int dr = pmtp_poll_draft(gi * GS + t, false);+ if (dr < 0) return false;+ if (dr == 0) { ready = false; break; }+ }+ if (!ready) continue;+ }+ pick = gi;}if (pick < 0) continue;rr = pick + 1;@@@ -12351,6 -12807,18 +13075,19 @@@}}if (drive.d.failed) { std::printf("ERR %s\n", drive.d.fail ? drive.d.fail : "the expert pool failed"); return false; }+ if (pa_due() && !pipe_inflight()) { // STRATA_PIPE_ADAPT_EVERY: the pipeline is empty - one round, as batch_step++ if (pmtp && !pmtp_settle()) return false; // STRATA_PIPED_MTP: the queued commits and draft chains are done first+ const Clock::time_point ta = Clock::now();+ pa_last = pa_windows;+ if (!adapt()) {+ std::printf("ERR an adaptive refill failed\n");+ return false;+ }+ ++pa_rounds;+ pa_swaps += (int64_t) pending.size();+ apply_pending(true); // the copies land and the tables follow (the next launch plans from them)+ pa_ms += std::chrono::duration<double, std::milli>(Clock::now() - ta).count();+ }if (batch_telemetry_enabled) {const auto sample = stage_in_flight.sample(Clock::now(), !pipe_inflight() && !batch_on());if (sample) {@@@ -12360,16 -12828,17 +13097,22 @@@std::fflush(stdout);}}+ // every slot done: the last commits have run before anything reads a slot (between windows nothing does)+ if (pmtp && !pipe_inflight() && !batch_on() && !pmtp_settle()) return false;if (!pipe_inflight() && !batch_on() && bt_windows > 0) { // all idle: the timing line, as batch_stepconst double w = (double) bt_windows, wall = std::chrono::duration<double, std::milli>(Clock::now() - bt_start).count();+ char pa_txt[160] = "";+ if (pipe_adapt_every > 0)+ std::snprintf(pa_txt, sizeof pa_txt, "; adaptive tier (STRATA_PIPE_ADAPT_EVERY): %lld rounds, %lld experts "+ "swapped in, %.0f ms", (long long) pa_rounds, (long long) pa_swaps, pa_ms);std::fprintf(stderr, "strata batch (pipelined, %d groups of %d): %lld group-steps, %lld rows in %.0f ms = "- "%.1f rows/s (admissions included)", o.batch_groups, GS, (long long) bt_windows,- (long long) bt_rows, wall, 1000.0 * bt_rows / std::max(wall, 1e-9));- "%.1f rows/s (admissions included)%s\n", o.batch_groups, GS, (long long) bt_windows,++ "%.1f rows/s (admissions included)%s", o.batch_groups, GS, (long long) bt_windows,+ (long long) bt_rows, wall, 1000.0 * bt_rows / std::max(wall, 1e-9), pa_txt);+ if (pmtp)- std::fprintf(stderr, ", %lld of %lld MTP proposals kept", (long long) bt_accepted, (long long) bt_tokens);++ std::fprintf(stderr, " (%lld of %lld MTP proposals kept)", (long long) bt_accepted, (long long) bt_tokens);+ std::fprintf(stderr, "\n");+ pa_rounds = pa_swaps = 0;+ pa_ms = 0;for (int k = 0; k < n_pipe; ++k) {const std::string pr = stage_verifier(k).profile_report();if (!pr.empty()) std::fprintf(stderr, "strata batch GPU stages, stage %d (ms/window):%s\n", k + 1, pr.c_str());