Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
serve: the serial decode timing line shows the adaptive waits in its sum (#1864)
SanghunYun2251053b427c
diff --git a/docs/DETAILS.md b/docs/DETAILS.mdindex f60429bf2..15f377c71 100644--- a/docs/DETAILS.md+++ b/docs/DETAILS.md@@ -300,6 +300,12 @@ decoding a little: use it to compare, not to measure speed. `STRATA_LOOKAHEAD_STprediction every 100 windows and costs CPU. `tools/replay_cache_policy.py` replays a `--dump-routing` trace through theadaptive tier's swap policy offline.+In the serial serve loop the timing line lists the wall-time parts in the order they run: `adapt wait + verify ++commit/emit + draft + adapt join + other`. `other` is the decode time minus the named parts, not a step of its own.+The times are host wall times, averaged over every decode window, and the bracketed parts of verify are already inside+verify. The pipelined decode keeps the older line (`ms/window = verify ...`) and batch slots are not covered, so a script+that reads this line must accept `ms/window = adapt wait` for a serial run.+### Draft layer| Option | Effect |diff --git a/src/program/generate.cpp b/src/program/generate.cppindex 29172c9bc..5decbae3c 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -15530,16 +15530,33 @@ int main(int argc, char** argv) {if (dec_timing && dec_windows > 0) {const DecSnap d1 = dec_snap();const double w = (double) dec_windows, L = (double) g.n_layers;- std::fprintf(stderr, "strata decode timing: %lld windows, avg T %.2f, %.2f tokens/window, %.2f ms/window = "- "verify %.2f (GPU-reach wait %.2f + per-layer host %.2f [plan %.2f actq %.2f jobs %.2f "- "CPU %.2f] + stage %.2f) + commit/emit %.2f + draft %.2f; per layer-window: CPU experts "- "%.2f (%.2f entries), VRAM hits %.2f, PCIe %.2f; adapt wait %.2f join %.2f, %.1f swaps/window\n",- (long long) dec_windows, dec_T / w, produced_n / w, decode_ms / w, dt_run / w,- (d1.wait - ds0.wait) / w, (d1.pool - ds0.pool) / w, (d1.plan - ds0.plan) / w,- (d1.actq - ds0.actq) / w, (d1.jobs - ds0.jobs) / w, (d1.run - ds0.run) / w,- (d1.host - ds0.host) / w, dt_commit / w, dt_draft / w, (d1.misses - ds0.misses) / (w * L),- (d1.entries - ds0.entries) / (w * L), (d1.hits - ds0.hits) / (w * L), (d1.pcie - ds0.pcie) / (w * L),- dt_adapt_wait / w, dt_adapt_join / w, (double) ((ajob ? a_swapped : adapt_nswaps) - dt_sw0) / w);+ if (!pl_ran) {+ const double dt_other = decode_ms - dt_adapt_wait - dt_run - dt_commit - dt_draft - dt_adapt_join;+ std::fprintf(stderr,+ "strata decode timing: %lld windows, avg T %.2f, %.2f tokens/window, %.2f ms/window = "+ "adapt wait %.2f + verify %.2f (GPU-reach wait %.2f + per-layer host %.2f "+ "[plan %.2f actq %.2f jobs %.2f CPU %.2f] + stage %.2f) + commit/emit %.2f + "+ "draft %.2f + adapt join %.2f + other %.2f; per layer-window: CPU experts "+ "%.2f (%.2f entries), VRAM hits %.2f, PCIe %.2f; %.1f swaps/window\n",+ (long long) dec_windows, dec_T / w, produced_n / w, decode_ms / w, dt_adapt_wait / w,+ dt_run / w, (d1.wait - ds0.wait) / w, (d1.pool - ds0.pool) / w,+ (d1.plan - ds0.plan) / w, (d1.actq - ds0.actq) / w, (d1.jobs - ds0.jobs) / w,+ (d1.run - ds0.run) / w, (d1.host - ds0.host) / w, dt_commit / w, dt_draft / w,+ dt_adapt_join / w, dt_other / w, (d1.misses - ds0.misses) / (w * L),+ (d1.entries - ds0.entries) / (w * L), (d1.hits - ds0.hits) / (w * L),+ (d1.pcie - ds0.pcie) / (w * L), (double) ((ajob ? a_swapped : adapt_nswaps) - dt_sw0) / w);+ } else { // Keep the pipeline's existing timing format.+ std::fprintf(stderr, "strata decode timing: %lld windows, avg T %.2f, %.2f tokens/window, %.2f ms/window = "+ "verify %.2f (GPU-reach wait %.2f + per-layer host %.2f [plan %.2f actq %.2f jobs %.2f "+ "CPU %.2f] + stage %.2f) + commit/emit %.2f + draft %.2f; per layer-window: CPU experts "+ "%.2f (%.2f entries), VRAM hits %.2f, PCIe %.2f; adapt wait %.2f join %.2f, %.1f swaps/window\n",+ (long long) dec_windows, dec_T / w, produced_n / w, decode_ms / w, dt_run / w,+ (d1.wait - ds0.wait) / w, (d1.pool - ds0.pool) / w, (d1.plan - ds0.plan) / w,+ (d1.actq - ds0.actq) / w, (d1.jobs - ds0.jobs) / w, (d1.run - ds0.run) / w,+ (d1.host - ds0.host) / w, dt_commit / w, dt_draft / w, (d1.misses - ds0.misses) / (w * L),+ (d1.entries - ds0.entries) / (w * L), (d1.hits - ds0.hits) / (w * L), (d1.pcie - ds0.pcie) / (w * L),+ dt_adapt_wait / w, dt_adapt_join / w, (double) ((ajob ? a_swapped : adapt_nswaps) - dt_sw0) / w);+ }if (aov)std::fprintf(stderr, "strata decode adapt overlap: %lld windows took swaps still landing, a layer waited %.2f ms in all so far\n",(long long) aov_flips, ver.ms_gate);