Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Merge f-145-int (477ac13cb, the 0.1.45 release tree) into f-split-145

Niko12219a58a4b73a4f
diff --cc CMakeLists.txt
index 1066f9814,d900326e3..5fcb7f917
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@@ -1690,8 -1751,9 +1813,9 @@@ if(STRATA_NATIVE_EXPERTS
src/prefill/moe_mmq.cu
src/prefill/moe_fused.cu
src/prefill/moe_fused_iq.cu
+ src/prefill/moe_f16_kq.cu
src/prefill/ggml_cuda_host.cu
- "${_strata_mmq}/quantize.cu")
+ "${_strata_quantize_cu}")
if(STRATA_MMQ_KQUANTS)
# the same K-quant instances as the CUDA branch: on HIP they serve the dense GGUF projections
# of Gemm::native (STRATA_DENSE_MMQ=1), the mixed-quant packs' attention/PLE tensors
@@@ -1700,14 -1762,9 +1824,14 @@@
foreach(_t IN LISTS _strata_mmq_types)
list(APPEND _strata_mmq_sources "${_strata_mmq}/template-instances/mmq-instance-${_t}.cu")
endforeach()
+ if(_strata_mmq_tq_overlay)
+ foreach(_t tq2_t tqk6 tqk7)
+ list(APPEND _strata_mmq_sources "${_strata_tq_overlay_dir}/template-instances/mmq-instance-${_t}.cu")
+ endforeach()
+ endif()
set_source_files_properties(${_strata_mmq_sources} PROPERTIES LANGUAGE HIP)
# moe_fused.cu is Strata's own CUDA-shaped source: it takes the HIP compat headers, ggml's MMQ sources do not
- set_source_files_properties(src/prefill/moe_fused.cu src/prefill/moe_fused_iq.cu
+ set_source_files_properties(src/prefill/moe_fused.cu src/prefill/moe_fused_iq.cu src/prefill/moe_f16_kq.cu
PROPERTIES INCLUDE_DIRECTORIES "${STRATA_HIP_COMPAT_INCLUDE_DIR}")
add_library(strata_mmq STATIC ${_strata_mmq_sources})
target_include_directories(strata_mmq
diff --cc docs/DETAILS.md
index 39b29fc50,5c5724f12..b37012801
--- a/docs/DETAILS.md
+++ b/docs/DETAILS.md
@@@ -192,17 -214,11 +214,19 @@@ other ~18 GB), a 32 GB PC with a 12-16
layer has two (the share otherwise rounds to 0 at 1-2 misses). RTX 3090 + Ryzen 5 5600X: prompt read +5-7 % at 56K-117K
tokens; decode level alone, +2-3 % with `--adapt-async 1` and `--pcie-frac 0.6`
([report](../bench/results/2026-10-11-community-rtx3090-5600x-arena-pin-by-need/README.md)).
+- **Pinned-memory headroom** (Windows CUDA, on by default; `STRATA_PIN_RESERVE_MIB=N`, `0` = off, sets it on any OS): the arena registers
+ slice by slice until the driver refuses one, and on a 64 GB PC that left no page-locked memory at all for the few small buffers the
+ engine allocates after it (the verify window's staging: `verify: mapped staging allocation failed`, exit 1, on Swift 1.5 IQ2_XS). The
+ engine now holds 640 MiB (given back as soon as the arena is placed) and 160 MiB (kept until a required buffer is refused, or READY) while
+ the arena pins, so the arena stops about one layer earlier; a refused required buffer gives both back and tries once more. If that still
+ fails the error ends with `[pinned-memory-exhausted]` and names the `STRATA_ARENA_PIN_GIB` to start with. Linux has no such ceiling
+ (the NVIDIA driver pins past `ulimit -l`), so the reserve is off there. Test knobs: `STRATA_TEST_PIN_BUDGET_MIB=N` makes the engine's own
+ pinned allocations fail beyond N MiB, `STRATA_TEST_PIN_LEFT_KIB=N` lowers that ceiling after the arena to what is charged plus N KiB.
- `STRATA_EXCHANGE_ROTATE=1` (opt-in): an adaptive swap hands buffer ownership over instead of copying the evicted
- blob into the RAM copy (equal-size blobs, fully page-locked copy). Same tokens, fewer host copies; it works with
- `--adapt-async 1` too. Details and the measurement: [EXCHANGE_ROTATION.md](EXCHANGE_ROTATION.md).
+ blob into the RAM copy (fully page-locked copy; layers of different blob sizes keep spare buffers per size). Same
+ tokens, fewer host copies; it works with `--adapt-async 1` too. Swift 1.5 IQ3_S on an RTX 5090 with 64 GB RAM
+ (Windows, about 30 swaps per verify window): 164 -> 188 tok/s on one forced 1,500-token text, 155 -> 178 over 23
+ coding-agent turns. Details and the measurements: [EXCHANGE_ROTATION.md](EXCHANGE_ROTATION.md). Measured by us on an RTX 3060 12 GB (IQ3_XXS, resident RAM, default flags, 5 interleaved pairs): decode 58.96 -> 62.03 tok/s (+5.2%, 5/5 pairs), prompt speed unchanged; tokens identical with real adaptive swaps (Tesla P100, resident, blocking adaptation). It adds about 1.4 GiB of page-locked RAM, so leave it off when the RAM is already tight.
- The answers are the plain mapped mode's for the same expert placement: the bytes are the file's. With a page-locked
copy the GPU also takes its usual share of the misses over PCIe (`--pcie-frac`), as with enough RAM; `--pcie-frac 0`
(or `STRATA_RESIDENT_PIN=0`) gives the mapped mode's exact tokens.
diff --cc include/strata/kernels/iq_kernels.hpp
index bc5c5a04b,44e706bb0..ea2bdcf91
--- a/include/strata/kernels/iq_kernels.hpp
+++ b/include/strata/kernels/iq_kernels.hpp
@@@ -88,15 -84,27 +88,36 @@@ void native_grouped_set_v1(bool v1)
/// (0 all, 1 gate/up + SwiGLU + quantize, 2 down).
void native_expert_set_mode(int mode, int phase);
+/// APR (prism.hadamard): y = H_block (signs * x) per block of `block` values of each of `rows` rows of `width`
+/// (x == y allowed). `signs`: a device vector of `width` signs, or nullptr (identity). The same floats as the CPU's
+/// hadamard_rotate. The f16 form rotates in place, computing in float.
+void hadamard_rows(const float* x, float* y, int64_t rows, int64_t width, int block, const float* signs, void* stream);
+void hadamard_rows_f16(uint16_t* x, int64_t rows, int64_t width, int block, const float* signs, void* stream);
+/// A device copy (on the current device) of the host sign vector `host` of `width` signs, made on first use and
+/// kept for the process. Call it at setup, not inside a CUDA graph capture. nullptr for a nullptr `host`.
+const float* hadamard_device_signs(const float* host, int width);
+
+ /// EXPDEC (STRATA_EXPDEC=1, docs/EXPERTDEC.md): `native_expert_grouped` stages the group's q8_1 activations in shared
+ /// memory and computes several rows per lane, the same sums per (row, entry) as the default kernels (bitwise), for
+ /// gate/up IQ2_XXS / IQ2_XS / IQ3_XXS / IQ3_S / IQ2_S at n_embd 2560 and down IQ4_NL / Q2_0 at n_ff 640; other layouts
+ /// keep the default kernels. Set before graph capture; captured graphs keep the kernels they captured.
+ void native_expert_set_expdec(int on);
+ int native_expert_expdec();
+ /// Its knobs, read from the environment at startup (STRATA_EXPDEC_R 1 | 2, _DR 1 | 2 | 4, _MINB 2 | 4, _D20 0 | 1,
+ /// _RB, _DRB, _MINBLK): gate/up and down rows per lane, the gate/up kernel's __launch_bounds__ minimum blocks per SM
+ /// (2 = up to 128 registers, 4 = up to 64), EXPDEC for the IQ4_NL down too, gate/up and down rows per block (0 = auto
+ /// from the launch's groups), the auto rule's minimum blocks. In the setter a field <= 0 (< 0 for the two rows per
+ /// block and down_iq4_nl) keeps the current value.
+ /// `down` (STRATA_EXPDEC_DOWN, default 0): EXPDEC for the down too; by default only the gate/up (in the engine the
+ /// Q2_0 down was ~10% slower with EXPDEC, docs/EXPERTDEC.md par. 7).
+ struct NativeExpertExpdecParams {
+ int gu_rows_per_lane = 0, down_rows_per_lane = 0, gu_min_blocks_per_sm = 0, down_iq4_nl = -1, down = -1;
+ int gu_rows_per_block = -1, down_rows_per_block = -1, min_blocks = 0;
+ };
+ void native_expert_set_expdec_params(const NativeExpertExpdecParams& p);
+ NativeExpertExpdecParams native_expert_expdec_params();
+ /// Whether EXPDEC has kernels for this layout's gate/up and down (the bench's check).
+ bool native_expert_expdec_takes(const NativeExpertLayout& L, bool& gu, bool& down);
/// `iq_mmvq` and `native_expert_grouped` decode each weight part once and apply it to every column / entry;
/// true selects the older kernels that decode it again per column (STRATA_OLD_IQ_MMVQ=1 at startup). Both give
/// bitwise the same results. Set before graph capture; captured graphs keep the kernels they captured.
diff --cc serve/server.py
index f1edd6b9c,2e805773d..31dceb85b
--- a/serve/server.py
+++ b/serve/server.py
@@@ -7112,6 -6556,43 +7169,47 @@@ def clean_shared_defaults(d) -> dict
return out
+ def split_joined_args(cfg: dict) -> None:
+ """#2267: an engine option written as ONE element of the config's "args" - "--ple-io mmap" or "--ple-io=mmap" - reaches
+ the engine as a single argument ("unknown argument: --ple-io mmap"; the usual form is two elements, "--ple-io",
+ "mmap"). Recommend, never force: such an element is split into the flag and its value, and said at startup. Only an
+ element that starts with a flag name (--letters-digits-dashes) followed by a space or "=" is touched, and never one
+ that sits in the value place of the flag before it."""
+ args = cfg.get("args")
+ if not isinstance(args, list):
+ return
+ out, changed = [], []
+ for a in args:
+ m = re.match(r"^(--[A-Za-z0-9][A-Za-z0-9-]*)(?:\s+|=)(\S.*)$", a) if isinstance(a, str) else None
+ prev = out[-1] if out else None
+ in_value_place = isinstance(prev, str) and re.match(r"^--[A-Za-z0-9][A-Za-z0-9-]*$", prev) is not None
+ if m is None or in_value_place:
+ out.append(a)
+ continue
+ out += [m.group(1), m.group(2).strip()]
+ changed.append(a)
+ if changed:
+ cfg["args"] = out
+ print("[strata] the config's args hold " + ", ".join(repr(c) for c in changed) + " as one element each: split into the "
+ 'flag and its value (write them as two elements, e.g. "--ple-io", "mmap")', flush=True)
+
+
+ def read_config(path) -> dict:
+ """The run config's JSON. A mistake in it (a missing comma between two args, a trailing comma) is said with its
+ place in the file instead of a traceback (#2267: editing "args" by hand is where it happens)."""
- text = Path(path).read_text(encoding="utf-8-sig") # Notepad adds a BOM
++ return parse_config_text(Path(path).read_text(encoding="utf-8-sig"), path) # Notepad adds a BOM
++
++
++def parse_config_text(text: str, path) -> dict:
++ """read_config on text already read (the owner-launched server hashes the bytes it parses, launch attestation)."""
+ try:
+ return json.loads(text)
+ except json.JSONDecodeError as e:
+ raise SystemExit(f"[strata] {path} is not valid JSON: {e.msg} at line {e.lineno}, column {e.colno}. Check the commas "
+ 'between the entries of "args" (each option and each value is its own quoted entry, e.g. '
+ '"--ple-io", "mmap",) and that no entry is followed by a comma at the end of a list.') from None
+
+
def lift_reasoning_effort_arg(cfg: dict) -> None:
"""#1641: "--reasoning-effort high" in a config's "args" is meant for the thinking level, which is not an engine
option (the engine stopped at the unknown flag). It is the config's "reasoning_effort" (none, low, medium or high:
@@@ -7461,71 -6845,9 +7559,72 @@@ def main(argv=None, *, lifecycle_event=
help="data-parallel replicas (opt-in): cut the \"gpu\" list into N equal groups and run one engine on "
"each, requests spread over them (also \"replicas\" in the config: a number, or "
"[{\"gpus\": [0, 1]}, {\"gpus\": [2, 3]}])")
- a = ap.parse_args()
- cfg = read_config(a.config) if a.config else {}
+ launch_args = list(sys.argv[1:] if argv is None else argv)
+ a = ap.parse_args(launch_args)
+ # Every ordinary way of being asked to stop takes Ctrl+C's path (QUIT to the engine, then exit): SIGTERM (#96 docker
+ # stop; systemd), SIGHUP (the terminal closed, ssh dropped), and on Windows Ctrl+Break and the console's close,
+ # logoff and shutdown events. They are installed before anything slow starts, so one that arrives while the model
+ # loads is not the default "die at once". SIGINT keeps Python's own handler (a second Ctrl+C stops waiting).
+ termination = "sigint"
+ finished = threading.Event() # set when the clean stop is over (a Windows console handler waits for it)
+
+ def request_stop(reason: str):
+ nonlocal termination
+ termination = reason
+ if threading.current_thread() is threading.main_thread():
+ raise KeyboardInterrupt
+ import _thread
+ _thread.interrupt_main()
+ lifeline.install_stop_handlers(request_stop, finished)
+ if a.owner_fd is not None:
+ try:
+ owner_guard.start(a.owner_fd, grace=(a.shutdown_timeout or 0) + 40)
+ except (OSError, ValueError) as exc:
+ ap.error(str(exc))
+ if a.shutdown_timeout is not None and not 0 <= a.shutdown_timeout <= 300:
+ ap.error("--shutdown-timeout must be in 0..300 seconds")
+ config_bytes = Path(a.config).read_bytes() if a.config else b""
- cfg = json.loads(config_bytes.decode("utf-8-sig")) if a.config else {} # Notepad adds a BOM
++ cfg = parse_config_text(config_bytes.decode("utf-8-sig"), a.config) if a.config else {} # Notepad adds a BOM
+ split_joined_args(cfg)
+ if a.config and a.engine == "strata":
+ cfg = apply_saved_tuning(cfg)
+ attestation_key = os.environ.pop("STRATA_OWNER_ATTEST_KEY", None)
+ launch_attestation = None
+ if a.owner_fd is not None:
+ if attestation_key is None or not re.fullmatch(r"[0-9a-f]{64}", attestation_key):
+ ap.error("owned server needs a private launch attestation key")
+ launch_attestation = launch_identity.attestation(bytes.fromhex(attestation_key),
+ config_bytes, launch_args)
+ if "batch_telemetry" in cfg and type(cfg["batch_telemetry"]) is not bool:
+ ap.error("batch_telemetry must be true or false")
+ try:
+ ready_path = ready_file.preflight(a.ready_file) if a.ready_file else None
+ except ValueError as e:
+ ap.error(str(e))
+ max_queue = a.max_queue if a.max_queue is not None else cfg.get("max_queue")
+ if max_queue is not None and (isinstance(max_queue, bool) or not isinstance(max_queue, int)
+ or not 0 <= max_queue <= 4096):
+ ap.error("--max-queue / max_queue must be a whole number in 0..4096")
+ queue_timeout = a.queue_timeout if a.queue_timeout is not None else cfg.get("queue_timeout")
+ if queue_timeout is not None and (isinstance(queue_timeout, bool) or not isinstance(queue_timeout, (int, float))
+ or not math.isfinite(queue_timeout) or not 0.1 <= queue_timeout <= 600):
+ ap.error("--queue-timeout / queue_timeout must be finite seconds in 0.1..600")
+ if queue_timeout is not None and max_queue is None:
+ ap.error("--queue-timeout requires --max-queue / max_queue to bound pending calls")
+ if a.parallel is not None:
+ if a.parallel < 1 or a.parallel > 1024:
+ ap.error("--parallel must be in 1..1024; the engine may reduce it to the supported count")
+ if "--batch" in cfg.get("args", []) or "--slots" in cfg.get("args", []):
+ ap.error("--parallel conflicts with an explicit --batch/--slots in the saved engine args")
+ cfg["parallel"] = a.parallel
+ if a.batch_groups is not None:
+ if a.batch_groups != "auto" and (not a.batch_groups.isdecimal() or int(a.batch_groups) < 1):
+ ap.error("--batch-groups must be auto or a positive integer")
+ if not cfg or "args" not in cfg:
+ ap.error("--batch-groups needs an engine config")
+ if "--batch-groups" in cfg["args"]:
+ ap.error("--batch-groups conflicts with the saved engine args")
+ cfg["args"] = [*cfg["args"], "--batch-groups", a.batch_groups]
lift_reasoning_effort_arg(cfg)
if a.replicas is not None:
cfg["replicas"] = a.replicas
diff --cc src/core/layer.cpp
index fa421e4e2,701ef22eb..dd985dfae
--- a/src/core/layer.cpp
+++ b/src/core/layer.cpp
@@@ -517,6 -520,53 +521,53 @@@ namespace
int64_t g_kv_resident = 0;
uint64_t g_kv_host_bytes = 0;
+ // The KV host copy: pinned and device-mapped. cudaHostAlloc is the default, but under WSL2 the driver caps what
+ // cudaHostAlloc(Mapped) may pin at about 1 GiB in all, while cudaHostRegister of ordinary memory is not capped there:
+ // the expert arena registers tens of GiB the same way (pinned.cu). So under WSL2, detected as generate.cpp's
+ // under_wddm() does (/dev/dxg, the device its GPU goes through to the Windows driver), the copy is anonymous memory
+ // registered Portable|Mapped before any page is touched, which also leaves WSL's small cudaHostAlloc budget to the
+ // staging buffers. Elsewhere cudaHostAlloc stays first. Either way the other method is the fallback when the first
+ // is refused. STRATA_KV_HOST_REGISTER=1/0 forces registering first or cudaHostAlloc first. Like the cudaHostAlloc
+ // copy it lives as long as the process.
+ uint8_t* kv_host_alloc(uint64_t bytes) {
+ auto host_alloc = [](uint64_t n) -> uint8_t* {
+ uint8_t* h = nullptr;
- if (cudaHostAlloc((void**) &h, n, cudaHostAllocMapped | cudaHostAllocPortable) == cudaSuccess) return h;
++ if (strata::core::host_alloc_guarded((void**) &h, n, cudaHostAllocMapped | cudaHostAllocPortable) == cudaSuccess) return h;
+ (void) cudaGetLastError();
+ return nullptr;
+ };
+ #ifdef _WIN32
+ return host_alloc(bytes);
+ #else
+ static const bool register_first = [] {
+ const char* e = std::getenv("STRATA_KV_HOST_REGISTER");
+ if (e != nullptr && (e[0] == '0' || e[0] == '1')) return e[0] == '1';
+ return std::filesystem::exists("/dev/dxg");
+ }();
+ auto register_anon = [](uint64_t n) -> uint8_t* {
+ void* m = mmap(nullptr, (size_t) n, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ if (m == MAP_FAILED) return nullptr;
+ if (cudaHostRegister(m, (size_t) n, cudaHostRegisterPortable | cudaHostRegisterMapped) != cudaSuccess) {
+ (void) cudaGetLastError();
+ munmap(m, (size_t) n);
+ return nullptr;
+ }
+ static bool said = false;
+ if (!said) {
+ std::fprintf(stderr, "strata: KV streaming: the host copy is pinned with cudaHostRegister\n");
+ said = true;
+ }
+ return (uint8_t*) m;
+ };
+ if (register_first) {
+ if (uint8_t* m = register_anon(bytes)) return m;
+ return host_alloc(bytes);
+ }
+ if (uint8_t* h = host_alloc(bytes)) return h;
+ return register_anon(bytes);
+ #endif
+ }
+
// ---- the elastic K/V (--kv-grow): one VMM range per state, each pool array at a chunk boundary in it
bool g_kv_elastic = false;
int64_t g_kv_elastic_init = 16384;
diff --cc src/core/mtp.cpp
index 2dbe1a67b,68be44206..58627029d
--- a/src/core/mtp.cpp
+++ b/src/core/mtp.cpp
@@@ -1,6 -1,6 +1,7 @@@
// src/core/mtp.cpp - see include/strata/core/mtp.hpp.
#include "strata/core/mtp.hpp"
+#include "strata/core/pin_budget.hpp"
+ #include "strata/kernels/qsa_prompt_attn.hpp"
#include "strata/core/coupled_draft.hpp"
#include "strata/core/spec_prob.hpp"
#include "strata/core/on_device.hpp"
diff --cc src/core/peer_experts.cpp
index 5a2164ca1,ee0c2d0c3..0320b75e5
--- a/src/core/peer_experts.cpp
+++ b/src/core/peer_experts.cpp
@@@ -1,8 -1,8 +1,9 @@@
// src/core/peer_experts.cpp - see include/strata/core/peer_experts.hpp.
#include "strata/core/peer_experts.hpp"
+ #include "strata/core/device.hpp" // primary_device() (STRATA_PRIMARY_DEVICE)
#include "strata/kernels/cpu/expert_layout.hpp"
+#include "strata/kernels/cpu/hadamard.hpp"
#include "strata/kernels/elementwise.hpp"
#include "strata/kernels/iq_kernels.hpp"
#include "strata/kernels/quantize_act.hpp"
diff --cc src/core/verify.cpp
index 59d4fe63d,64ffc92a2..86db7e3fa
--- a/src/core/verify.cpp
+++ b/src/core/verify.cpp
@@@ -1,8 -1,8 +1,9 @@@
// src/core/verify.cpp - see include/strata/core/verify.hpp.
#include "strata/core/batch_rows.hpp"
+ #include "strata/core/sm75_series.hpp"
#include "strata/kernels/gfx_arch.hpp"
#include "strata/core/verify.hpp"
+#include "strata/core/pin_budget.hpp"
#if !defined(STRATA_USE_HIP) && !defined(STRATA_HIP_GFX906)
#include "strata/core/host_graph_launch.hpp"
#endif
@@@ -664,7 -720,18 +722,18 @@@ bool Verifier::init(const WeightTable&
mapped(64, (void**) &h_flagB_, (void**) &m_flagB_) &&
mapped(64, (void**) &h_plan_err_, (void**) &m_plan_err_) &&
mapped(T * K * N * 4, (void**) &h_ymiss_, (void**) &m_ymiss_);
- if (!ok) { err = "verify: mapped staging allocation failed"; return false; }
+ if (!ok) { err = "verify: mapped staging allocation failed " + strata::core::pin_exhausted_hint(); return false; }
+ if (bal_ != nullptr) { // set_pcie_balance: four stamp words per (layer, group)
+ if (!mapped((size_t) g.n_layers * 2 * 4 * 8, (void**) &h_bal_, (void**) &m_bal_)) {
+ err = "verify: mapped balance stamps allocation failed";
+ return false;
+ }
+ bal_p_.assign((size_t) g.n_layers * 2, 0.0);
+ bal_nc_.assign((size_t) g.n_layers * 2, 0);
+ bal_np_.assign((size_t) g.n_layers * 2, 0);
+ bal_ok_.assign((size_t) g.n_layers * 2, 0);
+ bal_->on = true;
+ }
// the GPU plan: counts(4) | start(cap+1) | dst(cap) | tok(cap) | pad | ptr(cap u64) | ptr2(cap u64) | start2(cap+1)
{
const int64_t cap = (int64_t) (T * K);
diff --cc src/prefill/prefill.cpp
index 0f68a3002,17428925e..56ae3b358
--- a/src/prefill/prefill.cpp
+++ b/src/prefill/prefill.cpp
@@@ -3691,16 -3802,14 +3870,22 @@@ bool Prefill::run_impl(const int64_t* t
const size_t mmq_gub = use_mmq ? mmq::matrix_bytes(mmq_gt, 1280, N) : 0;
const size_t mmq_db = use_mmq ? mmq::matrix_bytes(mmq_dt, N, 640) : 0;
pt.mark(kPfGather, cs);
+ // the routed experts' input: `mixed`, or (APR) its rotated copy, which the peer / the layer
+ // split's helper also take (one rotation for both cards; `mixed` itself feeds the shared expert)
+ const float* xsrc = m.mixed;
if (use_mmq) {
- // step 2b: the layer's activations as q8_1 rows in expert order, straight from `mixed`
- // step 2b: the layer's activations as q8_1 rows in expert order, straight from `mixed`:
- // each token quantized once and written to its K rows (slot_dev, the inverse of src_dev).
++ // step 2b: the layer's activations as q8_1 rows in expert order, straight from `mixed` (APR:
++ // from its rotated copy `Xr`): each token quantized once and written to its K rows
++ // (slot_dev, the inverse of src_dev).
+ // NVFP4 w4a4 activations carry a per-row scale (Xscale): that mode keeps the gather.
+ if (had_gu) {
+ strata::kernels::hadamard_rows(m.mixed, m.Xr, T, N, had_block, had_sx, m.cs);
+ xsrc = m.Xr;
+ }
- mmq::quantize(xsrc, m.src_dev, m.Xq, mmq_gt, N, N, T * K, m.cs, m.Xscale);
+ if (mmq::fp4_activations(mmq_gt))
- mmq::quantize(m.mixed, m.src_dev, m.Xq, mmq_gt, N, N, T * K, m.cs, m.Xscale);
++ mmq::quantize(xsrc, m.src_dev, m.Xq, mmq_gt, N, N, T * K, m.cs, m.Xscale);
+ else
- mmq::quantize_scatter(m.mixed, m.slot_dev, m.src_dev, m.Xq, mmq_gt, N, N, T, (int) K, m.cs);
++ mmq::quantize_scatter(xsrc, m.slot_dev, m.src_dev, m.Xq, mmq_gt, N, N, T, (int) K, m.cs);
// each group's rows: absolute bounds (gate/up reads the layer's rows), relative ones (down
// reads the group's own quantized H)
const size_t n = order.size(), ng = (n + MMQ_GROUP - 1) / MMQ_GROUP;
diff --cc src/program/generate.cpp
index cdfaf1103,aae516316..3035bf166
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@@ -10759,7 -11228,12 +11286,13 @@@ int main(int argc, char** argv)
}
G.inflight = true;
G.stage = 0;
+ if (batch_telemetry_enabled) G.started = Clock::now();
+ // (with the doorbells on their own thread, a failure it set for a window that left this stage is
+ // not cleared unseen)
+ if (conc_serving && drive.d.failed) {
+ std::printf("ERR %s\n", drive.d.fail ? drive.d.fail : "the expert pool failed");
+ return false;
+ }
drive.d.failed = false;
apply_pending(false);
}
@@@ -10771,17 -11244,15 +11304,24 @@@
return false;
}
stage_group[(size_t) k] = pick;
+ if (conc_serving) { // the doorbell thread may be waiting between windows: the launch above is seen
+ std::atomic_thread_fence(std::memory_order_seq_cst);
+ if (bell_waiting.load(std::memory_order_relaxed)) {
+ std::lock_guard<std::mutex> lk(bell_mu);
+ bell_cv.notify_one();
+ }
+ }
}
if (drive.d.failed) { std::printf("ERR %s\n", drive.d.fail ? drive.d.fail : "the expert pool failed"); return false; }
+ if (batch_telemetry_enabled) {
+ const auto sample = stage_in_flight.sample(Clock::now(), !pipe_inflight() && !batch_on());
+ if (sample) {
+ std::printf("PSTAT %zu %lld", sample->in_flight_us.size(), (long long) sample->elapsed_us);
+ for (const int64_t busy : sample->in_flight_us) std::printf(" %lld", (long long) busy);
+ std::printf("\n");
+ std::fflush(stdout);
+ }
+ }
if (!pipe_inflight() && !batch_on() && bt_windows > 0) { // all idle: the timing line, as batch_step
const double w = (double) bt_windows, wall = std::chrono::duration<double, std::milli>(Clock::now() - bt_start).count();
std::fprintf(stderr, "strata batch (pipelined, %d groups of %d): %lld group-steps, %lld rows in %.0f ms = "
diff --cc tools/calibrate.py
index ca269d978,397cd8f1f..182c5a4d2
--- a/tools/calibrate.py
+++ b/tools/calibrate.py
@@@ -168,10 -222,8 +240,11 @@@ def run(cfg: dict, say=print, start_eng
tok = ST.Tokenizer(toks, (tpath / "merges.txt").read_text(encoding="utf-8").split("\n"),
json.loads((tpath / "token_type.json").read_text()))
ids_list = [chat_ids(tok, p) for p in PROMPTS]
+ if progress is None and cancel_event is None:
- return measure(engine_args(cfg), ids_list, start_engine, say, host_worker_extras())
- return measure(engine_args(cfg), ids_list, start_engine, say, host_worker_extras(),
- progress=progress, cancel_event=cancel_event)
++ return measure(engine_args(cfg), ids_list, start_engine, say, host_worker_extras(), doc_prompt(tok),
++ hip=cfg.get("backend") == "hip")
+ return measure(engine_args(cfg), ids_list, start_engine, say, host_worker_extras(), doc_prompt(tok),
- hip=cfg.get("backend") == "hip")
++ hip=cfg.get("backend") == "hip", progress=progress, cancel_event=cancel_event)
def engine_args(cfg: dict) -> list[str]:
@@@ -227,12 -272,9 +300,12 @@@ def sweep(s, values: list, base_tune: d
return out
- def measure(base_args: list[str], ids_list, start_engine, say=print, extra_workers=(),
-def measure(base_args: list[str], ids_list, start_engine, say=print, extra_workers=(), long_prompt=None, hip: bool = False) -> dict:
++def measure(base_args: list[str], ids_list, start_engine, say=print, extra_workers=(), long_prompt=None, hip: bool = False,
+ *, progress=None, cancel_event=None) -> dict:
t0 = time.time()
report: dict = {}
+ _check_cancel(cancel_event)
+ _progress(progress, "loading", 0, 1)
say(" Loading the model for the measurements ...")
base_args = apply(base_args, {}) # the product defaults: what the measurements must beat
eng = start_engine(base_args)
@@@ -241,12 -283,21 +314,22 @@@
d_pcie = float(info.get("pcie_frac", 0.55))
d_minp = float(info.get("spec_min_p", 0.5))
d_workers = int(info.get("pool_workers", 0)) or None
- s = Session(eng, ids_list)
+ s = Session(eng, ids_list, cancel_event)
s.warm_up()
+ _progress(progress, "loading", 1, 1)
+ # 0. a layer split's prompt chunks (prompt read speed; independent of the decode settings below)
+ pipe = 0
+ # the engine's own count of prompt stages where it reports one (a split can leave one, e.g.
+ # --split-skip-if-fits), else the flag
+ stages = info.get("prefill_stages")
+ split = stages > 1 if isinstance(stages, int) else "--layer-split" in base_args
+ if long_prompt is not None and split:
+ say(" Measuring the prompt chunks of the layer split ...")
+ pipe, rep_pipe = measure_pipe(eng, long_prompt, say)
+ report.update(rep_pipe)
# 1. the PCIe share, at the default draft floor
by_pcie = sweep(s, sorted(set(PCIE_FRACS) | {round(d_pcie, 2)}), {"spec_min_p": d_minp}, "pcie_frac",
- "PCIe share", say)
+ "PCIe share", say, progress, "pcie_share")
best_pcie = max(by_pcie, key=lambda k: statistics.median(by_pcie[k]))
# 2. the draft floor, at that share
by_minp = sweep(s, sorted(set(SPEC_MIN_PS) | {round(d_minp, 2)}), {"pcie_frac": best_pcie}, "spec_min_p",
@@@ -338,10 -375,34 +423,36 @@@
base_rate = statistics.median(by_adapt[a_best])
elif by_adapt.get("default"):
base_rate = statistics.median(by_adapt["default"])
+ # 6. HIP: the shared-expert stream fork (a restart each), with everything chosen so far. Both values are set
+ # explicitly, so a value the config carried before does not decide the baseline.
+ env = {}
+ if hip:
+ tuned = apply(base_args, settings)
+ by_fork = {}
+ for v in ("0", "1"):
++ _check_cancel(cancel_event)
+ say(f" Measuring the shared expert {'on its own stream' if v == '1' else 'on the main stream (the AMD default)'}"
+ " (restarts the engine) ...")
+ e = restart(lambda a: start_engine(a, {SH_STREAM: v}), tuned, f"{SH_STREAM}={v}", say, report)
+ if e is None:
+ continue
+ try:
- sf = Session(e, ids_list)
++ sf = Session(e, ids_list, cancel_event)
+ sf.warm_up(1)
+ by_fork[v] = [sf.rate(), sf.rate()]
+ say(f" {statistics.median(by_fork[v]):.1f} tok/s")
+ finally:
+ close(e)
+ report["sh_stream"] = by_fork
+ if pick(by_fork, "0") == "1":
+ env[SH_STREAM] = "1"
+ base_rate = statistics.median(by_fork["1"])
+ elif by_fork.get("0"):
+ base_rate = statistics.median(by_fork["0"])
report["seconds"] = round(time.time() - t0)
report["tok_s"] = round(base_rate, 1) if base_rate else None
+ _check_cancel(cancel_event)
- return {"settings": settings, "report": report}
+ return {"settings": settings, "env": env, "report": report}
def restart(start_engine, args, what, say, report):