Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Adaptive tier: the engine chooses the asynchronous tier by measurement (arena, one CUDA GPU)

Niko1221b18700c8f7d1
diff --git a/CMakeLists.txt b/CMakeLists.txt
index acc764463..7dd95dfdf 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1538,6 +1538,9 @@ if(STRATA_BUILD_TESTS)
add_executable(arch_defaults_test src/core/arch_defaults_test.cpp src/core/arch_defaults.cpp)
target_include_directories(arch_defaults_test PRIVATE include)
add_test(NAME arch_defaults_test COMMAND arch_defaults_test)
+ add_executable(adapt_async_default_test src/core/adapt_async_default_test.cpp)
+ target_include_directories(adapt_async_default_test PRIVATE include)
+ add_test(NAME adapt_async_default_test COMMAND adapt_async_default_test)
add_executable(uma_cache_test src/core/uma_cache_test.cpp)
target_include_directories(uma_cache_test PRIVATE include)
add_test(NAME uma_cache_test COMMAND uma_cache_test)
diff --git a/include/strata/core/adapt_async_default.hpp b/include/strata/core/adapt_async_default.hpp
new file mode 100644
index 000000000..7e0987d5d
--- /dev/null
+++ b/include/strata/core/adapt_async_default.hpp
@@ -0,0 +1,121 @@
+// include/strata/core/adapt_async_default.hpp - the asynchronous adaptive tier (--adapt-async 1) as a measured, per-session choice.
+//
+// Measured: on an RTX PRO 5000 box (64 cores) with every expert in the RAM arena, `--adapt-async 1` gave decode story +5.6%,
+// code +5.1%, doc8k +10.9% (5/5 pairs faster in every class; the tier's wait per verify window fell from 1.4 ms to 0.14 ms).
+// On a CPU-bound box (RTX A4000, Xeon E5-2643 v3, 12 threads, CPU experts 35 of 55 ms per window) the same switch gave
+// story 0.99, code 1.00, doc8k 1.04 and a 4K-prompt decode 0.945: the helper thread competes with the expert pool. So it
+// is neither always on nor always off: where the engine could run it (CUDA, `--serve`, one GPU, the expert arena, no batch
+// slots, no layer split) it starts on the blocking tier, measures ~64 windows, and only when the tier's wait is a large part
+// of a window does it try the asynchronous tier for ~64 windows and go back to the blocking tier if the decode got slower by more than 2%.
+// Off switches: `--adapt-async 0`, `STRATA_ADAPT_ASYNC=0`. `--adapt-async 1` keeps its old meaning (always on).
+// Not bit-exact from run to run, like the adaptive tier itself. The identity gates run with `--adapt-every 100000` (no
+// adaptive tier), so they never reach this.
+#pragma once
+
+#include <cstdlib>
+
+namespace strata::core {
+
+struct AdaptAsyncAuto {
+ int flag = -1; ///< --adapt-async on the command line: -1 not given, else 0 / 1
+ int env = -1; ///< STRATA_ADAPT_ASYNC: -1 unset, else its value (0 = off)
+ bool cuda = false; ///< a CUDA build (the HIP builds keep it opt-in)
+ bool serve = false;
+ bool adaptive = false; ///< --adapt-every and --adapt-swaps both above 0
+ bool arena = false; ///< the experts come from the RAM arena (every expert in RAM)
+ bool multi_gpu = false; ///< a layer split
+ bool pipeline_windows = false;
+ bool batch = false; ///< --batch slots
+ bool peer = false; ///< --peer-device
+ bool remote_opt = false; ///< --remote-expert-opt
+ bool adapt_overlap = false; ///< the opt-in STRATA_ADAPT_OVERLAP tier, which excludes the asynchronous one
+};
+
+/// STRATA_ADAPT_ASYNC as the process sees it (-1 unset).
+inline int adapt_async_env() {
+ const char* v = std::getenv("STRATA_ADAPT_ASYNC");
+ return v == nullptr || v[0] == '\0' ? -1 : std::atoi(v);
+}
+
+/// true: the engine may choose the asynchronous tier by measurement. `why_not` (may be null) gets a short reason otherwise.
+inline bool adapt_async_self_decides(const AdaptAsyncAuto& a, const char** why_not = nullptr) {
+ auto no = [&](const char* why) { if (why_not != nullptr) *why_not = why; return false; };
+ if (a.flag >= 0) return no("--adapt-async was given"); // the user's choice, 0 or 1, is applied as before
+ if (a.env == 0) return no("STRATA_ADAPT_ASYNC=0");
+ if (!a.cuda) return no("not a CUDA build");
+ if (!a.serve) return no("it needs --serve");
+ if (!a.adaptive) return no("the adaptive tier is off");
+ if (!a.arena) return no("not the expert arena");
+ if (a.multi_gpu) return no("a layer split");
+ if (a.pipeline_windows) return no("--pipeline-windows");
+ if (a.batch) return no("--batch slots");
+ if (a.peer) return no("--peer-device");
+ if (a.remote_opt) return no("--remote-expert-opt");
+ if (a.adapt_overlap) return no("STRATA_ADAPT_OVERLAP");
+ return true;
+}
+
+/// The measure -> trial -> keep or go back decision. Feed it one finished decode window at a time (wall ms of the window,
+/// ms of it the adaptive tier held it up, tokens it produced); it answers what to do before the next window.
+class AdaptAsyncSelf {
+public:
+ enum class Phase { Off, Measure, Trial, Done };
+ enum class Action { None, ToAsync, ToBlocking };
+ struct Config {
+ int warmup_windows = 64; ///< steady windows ignored first: the swap tier, the cache and the drafter are still settling after a start
+ int measure_windows = 64; ///< windows measured on the blocking tier
+ int trial_windows = 64; ///< windows measured on the asynchronous tier (after trial_skip warm-up windows)
+ int trial_skip = 4;
+ double wait_frac_min = 0.05; ///< try the asynchronous tier when the wait is at least this part of a window
+ double gain_min = 0.98; ///< keep it unless ms per token got worse by more than 1/gain_min (a guard against harm: 64 windows of text cannot resolve a 3-5% gain)
+ };
+ AdaptAsyncSelf() = default;
+ explicit AdaptAsyncSelf(const Config& c) : c_(c) {}
+
+ void arm() { phase_ = Phase::Measure; reset_sums(); }
+ Phase phase() const { return phase_; }
+ bool trialing() const { return phase_ == Phase::Trial; }
+ bool kept() const { return phase_ == Phase::Done && kept_; }
+
+ /// One decode window. Windows with no token (or a non-positive time) are ignored.
+ Action window(double window_ms, double wait_ms, int tokens) {
+ if (tokens <= 0 || !(window_ms > 0.0)) return Action::None;
+ if (phase_ == Phase::Measure) {
+ if (warmed_ < c_.warmup_windows) { ++warmed_; return Action::None; }
+ ms_ += window_ms; wait_ += wait_ms; tok_ += tokens; ++n_;
+ if (n_ < c_.measure_windows) return Action::None;
+ measure_ms_per_token_ = ms_ / (double) tok_;
+ wait_frac_ = ms_ > 0.0 ? wait_ / ms_ : 0.0;
+ reset_sums();
+ if (wait_frac_ >= c_.wait_frac_min) { phase_ = Phase::Trial; skipped_ = 0; return Action::ToAsync; }
+ phase_ = Phase::Done; kept_ = false; // the wait is small: nothing to win
+ return Action::None;
+ }
+ if (phase_ == Phase::Trial) {
+ if (skipped_ < c_.trial_skip) { ++skipped_; return Action::None; }
+ ms_ += window_ms; tok_ += tokens; ++n_;
+ if (n_ < c_.trial_windows) return Action::None;
+ trial_ms_per_token_ = ms_ / (double) tok_;
+ phase_ = Phase::Done;
+ kept_ = trial_ms_per_token_ * c_.gain_min <= measure_ms_per_token_;
+ return kept_ ? Action::None : Action::ToBlocking;
+ }
+ return Action::None;
+ }
+
+ double wait_fraction() const { return wait_frac_; }
+ double measure_ms_per_token() const { return measure_ms_per_token_; }
+ double trial_ms_per_token() const { return trial_ms_per_token_; }
+
+private:
+ void reset_sums() { ms_ = wait_ = 0.0; tok_ = 0; n_ = 0; }
+ Config c_{};
+ Phase phase_ = Phase::Off;
+ double ms_ = 0.0, wait_ = 0.0;
+ long long tok_ = 0;
+ int n_ = 0, skipped_ = 0, warmed_ = 0;
+ double wait_frac_ = 0.0, measure_ms_per_token_ = 0.0, trial_ms_per_token_ = 0.0;
+ bool kept_ = false;
+};
+
+} // namespace strata::core
diff --git a/src/core/adapt_async_default_test.cpp b/src/core/adapt_async_default_test.cpp
new file mode 100644
index 000000000..96a748a0a
--- /dev/null
+++ b/src/core/adapt_async_default_test.cpp
@@ -0,0 +1,135 @@
+// adapt_async_default_test: the asynchronous adaptive tier as a measured choice (include/strata/core/adapt_async_default.hpp).
+#include "strata/core/adapt_async_default.hpp"
+
+#include <cstdio>
+#include <cstring>
+
+static int g_fail = 0;
+#define CHECK(c) \
+ do { \
+ if (!(c)) { std::fprintf(stderr, "FAIL %s:%d %s\n", __FILE__, __LINE__, #c); ++g_fail; } \
+ } while (0)
+
+using strata::core::AdaptAsyncAuto;
+using strata::core::AdaptAsyncSelf;
+using Action = AdaptAsyncSelf::Action;
+using Phase = AdaptAsyncSelf::Phase;
+
+static AdaptAsyncAuto good() {
+ AdaptAsyncAuto a;
+ a.cuda = a.serve = a.adaptive = a.arena = true;
+ return a;
+}
+
+static AdaptAsyncSelf::Config cfg0() { // no warm-up: the cases below count their windows from the first
+ AdaptAsyncSelf::Config c;
+ c.warmup_windows = 0;
+ return c;
+}
+
+// feed `n` windows of `ms` / `wait` / `tok`; returns the last non-None action
+static Action feed(AdaptAsyncSelf& s, int n, double ms, double wait, int tok) {
+ Action last = Action::None;
+ for (int i = 0; i < n; ++i) {
+ const Action a = s.window(ms, wait, tok);
+ if (a != Action::None) last = a;
+ }
+ return last;
+}
+
+int main() {
+ const char* why = nullptr;
+ // ---- where the engine may choose by measurement
+ CHECK(strata::core::adapt_async_self_decides(good(), &why));
+ { AdaptAsyncAuto a = good(); a.flag = 0; CHECK(!strata::core::adapt_async_self_decides(a, &why)); CHECK(why && std::strstr(why, "given")); }
+ { AdaptAsyncAuto a = good(); a.flag = 1; CHECK(!strata::core::adapt_async_self_decides(a)); }
+ { AdaptAsyncAuto a = good(); a.env = 0; CHECK(!strata::core::adapt_async_self_decides(a, &why)); CHECK(why && std::strstr(why, "STRATA_ADAPT_ASYNC")); }
+ { AdaptAsyncAuto a = good(); a.env = 1; CHECK(strata::core::adapt_async_self_decides(a)); }
+ { AdaptAsyncAuto a = good(); a.env = -1; CHECK(strata::core::adapt_async_self_decides(a)); }
+ { AdaptAsyncAuto a = good(); a.cuda = false; CHECK(!strata::core::adapt_async_self_decides(a)); }
+ { AdaptAsyncAuto a = good(); a.serve = false; CHECK(!strata::core::adapt_async_self_decides(a)); }
+ { AdaptAsyncAuto a = good(); a.adaptive = false; CHECK(!strata::core::adapt_async_self_decides(a)); } // --adapt-every 100000 gates
+ { AdaptAsyncAuto a = good(); a.arena = false; CHECK(!strata::core::adapt_async_self_decides(a)); }
+ { AdaptAsyncAuto a = good(); a.multi_gpu = true; CHECK(!strata::core::adapt_async_self_decides(a)); }
+ { AdaptAsyncAuto a = good(); a.pipeline_windows = true; CHECK(!strata::core::adapt_async_self_decides(a)); }
+ { AdaptAsyncAuto a = good(); a.batch = true; CHECK(!strata::core::adapt_async_self_decides(a)); }
+ { AdaptAsyncAuto a = good(); a.peer = true; CHECK(!strata::core::adapt_async_self_decides(a)); }
+ { AdaptAsyncAuto a = good(); a.remote_opt = true; CHECK(!strata::core::adapt_async_self_decides(a)); }
+ { AdaptAsyncAuto a = good(); a.adapt_overlap = true; CHECK(!strata::core::adapt_async_self_decides(a)); }
+ CHECK(!strata::core::adapt_async_self_decides(AdaptAsyncAuto{}));
+
+ // ---- the state machine
+ { // not armed: nothing happens
+ AdaptAsyncSelf s;
+ CHECK(s.phase() == Phase::Off);
+ CHECK(feed(s, 500, 20.0, 5.0, 3) == Action::None);
+ }
+ { // the wait is small (3% of a window): measure 64 windows, stay blocking, never switch
+ AdaptAsyncSelf s(cfg0()); s.arm();
+ CHECK(feed(s, 63, 20.0, 0.6, 3) == Action::None);
+ CHECK(s.phase() == Phase::Measure);
+ CHECK(feed(s, 1, 20.0, 0.6, 3) == Action::None);
+ CHECK(s.phase() == Phase::Done && !s.kept());
+ CHECK(feed(s, 500, 20.0, 0.6, 3) == Action::None);
+ }
+ { // the wait is 7%: switch to async after the 64th window; the async windows are 10% faster per token: kept
+ AdaptAsyncSelf s(cfg0()); s.arm();
+ CHECK(feed(s, 63, 20.0, 1.4, 3) == Action::None);
+ CHECK(s.window(20.0, 1.4, 3) == Action::ToAsync);
+ CHECK(s.phase() == Phase::Trial && s.trialing());
+ CHECK(s.wait_fraction() > 0.069 && s.wait_fraction() < 0.071);
+ CHECK(feed(s, 4 + 63, 18.0, 0.1, 3) == Action::None); // 4 warm-up windows are not counted
+ CHECK(s.window(18.0, 0.1, 3) == Action::None); // kept: no action
+ CHECK(s.phase() == Phase::Done && s.kept());
+ CHECK(s.trial_ms_per_token() < s.measure_ms_per_token());
+ }
+ { // the wait is 7% and async is 3% SLOWER per token (> 2%): back to blocking, and it stays there
+ AdaptAsyncSelf s(cfg0()); s.arm();
+ CHECK(feed(s, 64, 20.0, 1.4, 3) == Action::ToAsync);
+ CHECK(feed(s, 4 + 63, 20.6, 0.1, 3) == Action::None);
+ CHECK(s.window(20.6, 0.1, 3) == Action::ToBlocking);
+ CHECK(s.phase() == Phase::Done && !s.kept());
+ CHECK(feed(s, 1000, 19.6, 0.1, 3) == Action::None);
+ }
+ { // async slower (the CPU-bound box): back
+ AdaptAsyncSelf s(cfg0()); s.arm();
+ CHECK(feed(s, 64, 55.0, 4.0, 3) == Action::ToAsync);
+ CHECK(feed(s, 4 + 63, 58.0, 0.2, 3) == Action::None);
+ CHECK(s.window(58.0, 0.2, 3) == Action::ToBlocking);
+ }
+ { // async 1% slower or equal: kept (the trial guards against harm, it does not demand a gain)
+ AdaptAsyncSelf s(cfg0()); s.arm();
+ CHECK(feed(s, 64, 20.0, 1.4, 3) == Action::ToAsync);
+ CHECK(feed(s, 4 + 64, 20.2, 0.1, 3) == Action::None);
+ CHECK(s.kept());
+ }
+ { // 5% wait switches; a 3% gain is kept
+ AdaptAsyncSelf s(cfg0()); s.arm();
+ CHECK(feed(s, 64, 100.0, 5.0, 4) == Action::ToAsync);
+ const double t = 100.0 / 1.0301;
+ CHECK(feed(s, 4 + 64, t, 0.0, 4) == Action::None);
+ CHECK(s.kept());
+ }
+ { // windows with no token or no time are ignored (they neither count nor move the phase)
+ AdaptAsyncSelf s(cfg0()); s.arm();
+ CHECK(feed(s, 200, 20.0, 5.0, 0) == Action::None);
+ CHECK(feed(s, 200, 0.0, 0.0, 3) == Action::None);
+ CHECK(s.phase() == Phase::Measure);
+ }
+ { // tokens per window matter: more tokens per window at the same time is faster per token
+ AdaptAsyncSelf s(cfg0()); s.arm();
+ CHECK(feed(s, 64, 20.0, 2.0, 2) == Action::ToAsync); // 10 ms per token
+ CHECK(feed(s, 4 + 64, 20.0, 0.1, 3) == Action::None); // 6.7 ms per token: kept
+ CHECK(s.kept());
+ }
+ { // the default has a warm-up: the first 64 steady windows are not measured (a fresh engine's cache is still settling)
+ AdaptAsyncSelf s; s.arm();
+ CHECK(feed(s, 64, 20.0, 5.0, 3) == Action::None); // warm-up
+ CHECK(s.phase() == Phase::Measure);
+ CHECK(feed(s, 63, 20.0, 1.4, 3) == Action::None);
+ CHECK(s.window(20.0, 1.4, 3) == Action::ToAsync); // 128th window: the measurement is the last 64 only
+ CHECK(s.wait_fraction() > 0.069 && s.wait_fraction() < 0.071);
+ }
+ if (g_fail == 0) std::printf("adapt_async_default_test: OK\n");
+ return g_fail == 0 ? 0 : 1;
+}
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index 7bafd6850..d381227de 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -16,6 +16,7 @@
// (`phase-2-correct-engine.md:5-9`). What it is FOR is the honest tok/s figure and the logit dump.
#include "strata/platform/memory.hpp"
+#include "strata/core/adapt_async_default.hpp"
#include "strata/core/arch_defaults.hpp"
#include "strata/core/nv_defaults.hpp"
#include "strata/kernels/gfx_arch.hpp"
@@ -700,6 +701,8 @@ struct Options {
/// windows on a helper thread instead of one window waiting for a whole round (see the tier in the --serve block).
/// Not bit-exact run to run. 0 = the blocking tier.
int adapt_async = 0;
+ int adapt_async_given = -1; ///< --adapt-async on the command line (-1: not given; then the engine may decide, adapt_async_default.hpp)
+ bool adapt_async_self = false; ///< the engine measures and chooses the asynchronous tier during the session (adapt_async_default.hpp)
/// Plan v0.3 P6: a draft enters the verify window only while every draft before it (and itself) has at least
/// this probability under the draft layer; 0 = always --spec-1 drafts.
double spec_min_p = 0.0;
@@ -1071,10 +1074,12 @@ void usage() {
" does not fit. Same answers as --mmap-experts for the same placement.\n"
" --adapt-min-gain F the adaptive tier swaps an expert in only when its decayed routing count beats\n"
" the one it replaces by more than F (default 1.5; higher = fewer, surer swaps)\n"
- " --adapt-async 1 --serve with the resident RAM mode or the arena (one GPU; opt-in): the adaptive\n"
+ " --adapt-async 0|1 --serve with the resident RAM mode or the arena (one GPU): the adaptive\n"
" tier's swaps advance between verify windows instead of a window waiting for a\n"
- " whole round (the arena: copy-in only). Not bit-exact run to run. 0 (default)\n"
- " = the blocking tier.\n");
+ " whole round (the arena: copy-in only). Not bit-exact run to run. Without the flag,\n"
+ " the expert arena on one CUDA GPU (no --batch) starts blocking and tries it for a\n"
+ " while when the swaps hold the windows up, keeping it only if it is faster; 0 = the\n"
+ " blocking tier always (also STRATA_ADAPT_ASYNC=0); 1 = always on.\n");
}
// --lookup-chain: the context's last tokens for the draft sources (at most 64), the MTP's pending drafts last
@@ -2626,7 +2631,7 @@ int main(int argc, char** argv) {
else if (a == "--adapt-every") { o.adapt_every = std::atoi(next("--adapt-every")); o.adapt_every_given = true; }
else if (a == "--adapt-decay") o.adapt_decay = (float) std::atof(next("--adapt-decay"));
else if (a == "--adapt-min-gain") o.adapt_min_gain = (float) std::atof(next("--adapt-min-gain"));
- else if (a == "--adapt-async") o.adapt_async = std::atoi(next("--adapt-async")) != 0 ? 1 : 0;
+ else if (a == "--adapt-async") { o.adapt_async = std::atoi(next("--adapt-async")) != 0 ? 1 : 0; o.adapt_async_given = o.adapt_async; }
else if (a == "--spec-min-p") o.spec_min_p = std::atof(next("--spec-min-p"));
else if (a == "--stop-eos") o.stop_eos = true;
else if (a == "--spec-split") o.spec_split = true;
@@ -8074,6 +8079,27 @@ int main(int argc, char** argv) {
std::fprintf(stderr, "strata generate: --adapt-async 1 is off (%s): the blocking adaptive tier\n", why);
o.adapt_async = 0;
};
+ // Where the asynchronous tier could run on its own (CUDA, one GPU, the expert arena, no batch slots), the engine measures
+ // and chooses during the session: see include/strata/core/adapt_async_default.hpp. --adapt-async 0 or
+ // STRATA_ADAPT_ASYNC=0 keep the blocking tier; --adapt-async 1 keeps its old meaning.
+ if (o.adapt_async_given < 0) {
+ strata::core::AdaptAsyncAuto aa;
+ aa.flag = o.adapt_async_given;
+ aa.env = strata::core::adapt_async_env();
+#if !defined(STRATA_USE_HIP)
+ aa.cuda = true;
+#endif
+ aa.serve = o.serve;
+ aa.adaptive = o.adapt_every > 0 && o.adapt_swaps > 0;
+ aa.arena = srcp == &arena_src;
+ aa.multi_gpu = multi_gpu;
+ aa.pipeline_windows = o.pipeline_windows >= 2;
+ aa.batch = o.batch > 0;
+ aa.peer = peer.valid();
+ aa.remote_opt = remote_opt != nullptr;
+ aa.adapt_overlap = adapt_overlap();
+ o.adapt_async_self = strata::core::adapt_async_self_decides(aa);
+ }
if (o.adapt_async) {
static const bool pl_async = [] {
const char* v = std::getenv("STRATA_PIPELINE_ADAPT_ASYNC");
@@ -10541,15 +10567,25 @@ int main(int argc, char** argv) {
const bool a_inline = a_arena && a_env("STRATA_ADAPT_PICK_INLINE", 1) != 0;
const bool a_wait = a_arena && a_env("STRATA_ADAPT_ASYNC_WAIT", 1) != 0;
const size_t a_batch = a_arena ? (size_t) std::max(0, a_env("STRATA_ADAPT_ASYNC_BATCH", 0)) : 0;
- if (o.adapt_async && !drive.d.usage.empty() &&
+ // o.adapt_async_self (arena, one GPU, nothing explicit): the homes are set up, the job thread is created only when the
+ // session's measurement turns the asynchronous tier on (asw_*, in the decode loop)
+ const bool a_self = o.adapt_async_self && !o.adapt_async && a_arena && !drive.d.usage.empty();
+ if ((o.adapt_async || a_self) && !drive.d.usage.empty() &&
((src.complement_ready() && src.exchange_capacity() >= 2 * a_cap) || a_arena)) {
ahomes.push_back({&xcache, adapt_stream, adapt_ev, 0, false});
for (auto& st : stages) ahomes.push_back({&st->cache, st->adapt_stream, st->adapt_ev, st->dev, false});
- ajob = std::make_unique<JobThread>();
- std::fprintf(stderr, "strata serve: adaptive tier asynchronous (--adapt-async 1): up to %lld swaps per round, "
- "a round every %d windows%s\n", (long long) a_cap, o.adapt_every,
- a_arena ? " (arena: copy-in only)" : "");
+ if (o.adapt_async) {
+ ajob = std::make_unique<JobThread>();
+ std::fprintf(stderr, "strata serve: adaptive tier asynchronous (--adapt-async 1): up to %lld swaps per round, "
+ "a round every %d windows%s\n", (long long) a_cap, o.adapt_every,
+ a_arena ? " (arena: copy-in only)" : "");
+ } else {
+ std::fprintf(stderr, "strata serve: adaptive tier: blocking, and the asynchronous tier is tried once the session "
+ "shows the swaps hold the windows up (--adapt-async 0 or STRATA_ADAPT_ASYNC=0: never)\n");
+ }
}
+ strata::core::AdaptAsyncSelf asw; // the session's choice between the two tiers (adapt_async_default.hpp)
+ if (a_self && !ahomes.empty()) asw.arm();
// the pipelined loop's side (see above): whether stage `st` has no window in flight; per home, the copies back
// and in to queue at its next gap, the RAM fence still to pass and the gaps seen
std::function<bool(int)> a_pl_idle;
@@ -14934,6 +14970,8 @@ int main(int argc, char** argv) {
// expert ran on the GPU or the CPU (they round differently) depended on the copy's timing
// (STRATA_ADAPT_NOWAIT=1: 0.1.37's non-blocking query, the A/B)
const Clock::time_point taw0 = Clock::now();
+ const bool asw_skip = first_window || merged; // the session's tier choice counts steady decode windows only
+ double asw_wait_ms = 0;
if (adapt_nowait()) apply_pending(false);
else if (aov && !pending.empty() && !aflipped) {
// 0.1.43 STRATA_ADAPT_OVERLAP: the tables take the swaps now, exactly where the wait would have ended;
@@ -15062,7 +15100,8 @@ int main(int argc, char** argv) {
std::printf("ERR an adaptive refill failed\n");
return 1;
}
- dt_adapt_wait += std::chrono::duration<double, std::milli>(Clock::now() - taw0).count();
+ asw_wait_ms = std::chrono::duration<double, std::milli>(Clock::now() - taw0).count();
+ dt_adapt_wait += asw_wait_ms;
if (hist_n > 0) {
// the tails the penalties count over, ONE PER ROW: the tokens the state has consumed, the
// fed-back head `x` (it joins `consumed` only after this window commits), then the drafts
@@ -15177,12 +15216,36 @@ int main(int argc, char** argv) {
if (adapt_thr.joinable()) {
const Clock::time_point taj0 = Clock::now();
adapt_thr.join();
- dt_adapt_join += std::chrono::duration<double, std::milli>(Clock::now() - taj0).count();
+ const double asw_join = std::chrono::duration<double, std::milli>(Clock::now() - taj0).count();
+ dt_adapt_join += asw_join;
+ asw_wait_ms += asw_join;
}
if (!adapt_ok) {
std::printf("ERR an adaptive refill failed\n");
return 1;
}
+ // the session's choice of the adaptive tier (adapt_async_default.hpp): ~64 windows on the blocking tier, then
+ // ~64 on the asynchronous one if the swaps held the windows up, kept only if the decode got faster
+ if (!asw_skip && (asw.phase() == strata::core::AdaptAsyncSelf::Phase::Measure || asw.trialing())) {
+ const bool was_trial = asw.trialing();
+ const double asw_ms = std::chrono::duration<double, std::milli>(Clock::now() - taw0).count();
+ const auto act = asw.window(asw_ms, asw_wait_ms, a + 1);
+ if (act == strata::core::AdaptAsyncSelf::Action::ToAsync) {
+ ajob = std::make_unique<JobThread>();
+ std::fprintf(stderr, "strata serve: adaptive tier: the swaps held the windows up by %.1f%% (%.2f ms per "
+ "token): trying the asynchronous tier for a while\n",
+ 100.0 * asw.wait_fraction(), asw.measure_ms_per_token());
+ } else if (act == strata::core::AdaptAsyncSelf::Action::ToBlocking) {
+ adapt_quiesce();
+ ajob.reset();
+ std::fprintf(stderr, "strata serve: adaptive tier: the asynchronous tier was slower (%.2f against "
+ "%.2f ms per token): back to the blocking tier for this session\n",
+ asw.trial_ms_per_token(), asw.measure_ms_per_token());
+ } else if (was_trial && asw.kept()) {
+ std::fprintf(stderr, "strata serve: adaptive tier: the asynchronous tier stays on (%.2f against %.2f ms "
+ "per token)\n", asw.trial_ms_per_token(), asw.measure_ms_per_token());
+ }
+ }
if (!drafted) {
std::printf("ERR %s\n", err.c_str());
return 1;