Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Merge f-149-prefetch (82c80ac4d) into f-149-default (from f-split-int fb7c62963)

Niko1221240304928b59
diff --cc src/program/generate.cpp
index 52b76d18d,95975f1ad..7f565445a
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@@ -10991,21 -10775,9 +11007,22 @@@ int main(int argc, char** argv)
int64_t ba_window = 0, bt_adapt_rounds = 0, bt_adapt_swaps = 0;
double bt_adapt_wait = 0;
bool in_prompt_read = false; // batch_step runs between a prompt's chunks: the adaptive tier stands still then
+ double serve_adapt_drain_ms = 0.0; // 149: the request's wait for the adaptive round in flight
int64_t bt_a_rounds0 = 0, bt_a_swapped0 = 0; // the asynchronous tier's counters when the timed windows began
Clock::time_point bt_start = Clock::now();
+ const char* batch_telemetry_env = std::getenv("STRATA_BATCH_TELEMETRY");
+ const bool batch_telemetry_enabled = batch_telemetry_env != nullptr &&
+ batch_telemetry_env[0] == '1' && batch_telemetry_env[1] == '\0';
+ strata::core::BatchWindowSampler batch_telemetry(batch_telemetry_enabled);
+ auto report_batch_window = [&](int processed_rows, int active_rows, bool idle, Clock::time_point started) {
+ if (!batch_telemetry_enabled) return;
+ const auto sample = batch_telemetry.complete(processed_rows, active_rows, idle, started, Clock::now());
+ if (!sample) return;
+ std::printf("BSTAT %lld %lld %lld %lld\n", (long long) sample->windows,
+ (long long) sample->processed_rows, (long long) sample->active_rows,
+ (long long) sample->elapsed_ms);
+ std::fflush(stdout);
+ };
int admit_slot = -1; ///< the slot the request being read will continue in (BGEN)
long long admit_max_new = 0;
auto batch_on = [&] { for (const BSlot& b : bs) if (b.active) return true; return false; };