Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Build the engine for ARM64 (DGX Spark / GB10) without changing the x86 build (#3622)

ferstara4c41a92ee9e
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9c6f28790..57c6b4117 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -59,6 +59,17 @@ function(strata_utf8_manifest target)
endif()
endfunction()
+# The AVX kernels exist for x86 only. On aarch64 (DGX Spark / GB10) portable.cpp stands in for them and
+# native experts run on ggml-cpu's NEON dot products.
+if(NOT CMAKE_SYSTEM_PROCESSOR OR CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|amd64|x64|X64|i[3-6]86|x86|X86)$") # an unset processor stays the x86 build, as before
+ set(STRATA_CPU_X86 ON)
+else()
+ set(STRATA_CPU_X86 OFF)
+endif()
+if(STRATA_ISA_FLOOR AND NOT STRATA_CPU_X86)
+ message(WARNING "STRATA_ISA_FLOOR is an x86 older-CPU build and does not apply to ${CMAKE_SYSTEM_PROCESSOR}; ignored")
+ set(STRATA_ISA_FLOOR "" CACHE STRING "" FORCE)
+endif()
set(CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_EXTENSIONS OFF)
@@ -1644,8 +1655,13 @@ if(STRATA_BUILD_TESTS OR STRATA_BUILD_CONVERSATION_TESTS)
add_test(NAME batch_groups_test COMMAND batch_groups_test)
endif()
-add_library(strata_kernels_cpu STATIC src/kernels/cpu/expert.cpp src/kernels/cpu/pool.cpp
- src/kernels/cpu/expert_layout.cpp src/kernels/cpu/q2_avx2.cpp src/kernels/cpu/hadamard.cpp)
+if(STRATA_CPU_X86)
+ add_library(strata_kernels_cpu STATIC src/kernels/cpu/expert.cpp src/kernels/cpu/pool.cpp
+ src/kernels/cpu/expert_layout.cpp src/kernels/cpu/q2_avx2.cpp src/kernels/cpu/hadamard.cpp)
+else()
+ add_library(strata_kernels_cpu STATIC src/kernels/cpu/portable.cpp src/kernels/cpu/pool.cpp
+ src/kernels/cpu/expert_layout.cpp src/kernels/cpu/hadamard.cpp)
+endif()
target_include_directories(strata_kernels_cpu PUBLIC include)
# Plan v0.3 P6: only the kernel files get the AVX-512 / AVX2 flags; the pool and the probe run on any x86-64,
# and the engine picks the kernels at run time (cpu_avx512_ok).
@@ -1757,11 +1773,13 @@ if(STRATA_NATIVE_EXPERTS)
endif()
# part of strata_kernels_cpu (the pool calls these and they call its CPU probe: one library, so GNU ld's
# one-pass static linking has no cycle to resolve)
- target_sources(strata_kernels_cpu PRIVATE src/kernels/cpu/native_expert.cpp src/kernels/cpu/iq_avx512.cpp
- src/kernels/cpu/iq_avx2.cpp src/kernels/cpu/kq_avx2.cpp
- src/kernels/cpu/kq_avx1.cpp src/kernels/cpu/q2_avx1.cpp src/kernels/cpu/iq_avx1.cpp
- src/kernels/cpu/nvfp4_avx512.cpp src/kernels/cpu/nvfp4_avx2.cpp
- src/kernels/cpu/tq_cpu.cpp src/kernels/cpu/tq_avx2.cpp src/kernels/cpu/tq_avx512.cpp)
+ target_sources(strata_kernels_cpu PRIVATE src/kernels/cpu/native_expert.cpp)
+ if(STRATA_CPU_X86)
+ target_sources(strata_kernels_cpu PRIVATE src/kernels/cpu/iq_avx512.cpp src/kernels/cpu/iq_avx2.cpp
+ src/kernels/cpu/kq_avx2.cpp src/kernels/cpu/kq_avx1.cpp src/kernels/cpu/q2_avx1.cpp
+ src/kernels/cpu/iq_avx1.cpp src/kernels/cpu/nvfp4_avx512.cpp src/kernels/cpu/nvfp4_avx2.cpp
+ src/kernels/cpu/tq_cpu.cpp src/kernels/cpu/tq_avx2.cpp src/kernels/cpu/tq_avx512.cpp)
+ endif()
target_include_directories(strata_kernels_cpu PRIVATE third_party/ggml)
if(MSVC)
set_source_files_properties(src/kernels/cpu/iq_avx512.cpp src/kernels/cpu/nvfp4_avx512.cpp
diff --git a/include/strata/platform/cpu_relax.hpp b/include/strata/platform/cpu_relax.hpp
new file mode 100644
index 000000000..c5dba7d3f
--- /dev/null
+++ b/include/strata/platform/cpu_relax.hpp
@@ -0,0 +1,35 @@
+// include/strata/platform/cpu_relax.hpp - the spin-wait hint and the store fence, per architecture.
+//
+// x86: `_mm_pause` and `_mm_sfence` (the fence orders the write-combining stores into pinned host memory before
+// the doorbell the GPU polls). aarch64: `yield` and `dmb oshst` - the outer-shareable store barrier is what
+// makes host stores visible to a coherent device (GB10 / Grace NVLink-C2C) before a later store. Apple silicon (the
+// Metal build) keeps `dmb ishst`, the barrier its compat header (metal/include/metal_compat/immintrin.h) has always used.
+// Local port of the approach in https://github.com/Niko1221/Strata/pull/1077 onto main e8ca9af.
+#pragma once
+
+#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86)
+#include <immintrin.h>
+#define STRATA_X86 1
+// Macros on x86, so the code the compiler sees is exactly the intrinsic that stood there before (the objects of the
+// files that use them are identical before and after this header). x86 does not reorder a load before an older load:
+// the load fence has nothing to emit.
+#define strata_cpu_pause() _mm_pause()
+#define strata_store_fence() _mm_sfence()
+#define strata_load_fence() ((void) 0)
+#elif defined(__aarch64__)
+inline void strata_cpu_pause() { __asm__ __volatile__("yield" ::: "memory"); }
+#if defined(__APPLE__)
+inline void strata_store_fence() { __asm__ __volatile__("dmb ishst" ::: "memory"); }
+#define strata_load_fence() ((void) 0) // the Metal build has never had one
+#else
+inline void strata_store_fence() { __asm__ __volatile__("dmb oshst" ::: "memory"); }
+// After a spin saw a flag the GPU raised, the data the GPU wrote before it must be read after the flag: an ARM core
+// may otherwise load that data early. `dmb ishld` is the load barrier.
+inline void strata_load_fence() { __asm__ __volatile__("dmb ishld" ::: "memory"); }
+#endif
+#else
+#include <atomic>
+inline void strata_cpu_pause() {}
+inline void strata_store_fence() { std::atomic_thread_fence(std::memory_order_seq_cst); }
+inline void strata_load_fence() { std::atomic_thread_fence(std::memory_order_acquire); }
+#endif
diff --git a/setup.py b/setup.py
index 55f8941f2..241f68ee7 100644
--- a/setup.py
+++ b/setup.py
@@ -1335,7 +1335,11 @@ def cpu_info():
else:
try:
txt = open("/proc/cpuinfo").read()
- flags = set(re.search(r"^flags\s*:\s*(.*)$", txt, re.M).group(1).split())
+ mflags = re.search(r"^flags\s*:\s*(.*)$", txt, re.M)
+ if mflags is None:
+ # aarch64 lists Features, not the x86 flags line. The normal engine is the native one.
+ return name, False, False
+ flags = set(mflags.group(1).split())
avx2 = "avx2" in flags
avx512 = {"avx512f", "avx512bw", "avx512vl", "avx512_vnni", "avx512vbmi"} <= flags
m = re.search(r"^model name\s*:\s*(.*)$", txt, re.M)
@@ -1366,6 +1370,11 @@ def _cpuid_floor() -> str:
return ""
+def is_arm64() -> bool:
+ """An ARM64 CPU (DGX Spark / GB10, Grace): there is no AVX here, and no ready-made engine for it."""
+ return platform.machine().lower() in ("aarch64", "arm64")
+
+
def cpu_floor(avx2: bool) -> str:
"""The experimental older-CPU build this PC needs (#394 #595 #623): "" with AVX2 (the normal engine), "avx" (Sandy /
Ivy Bridge, AMD Bulldozer), "none" (SSE4.2 + POPCNT: Nehalem, Westmere), or "unsupported". STRATA_ISA_FLOOR=avx|
@@ -1373,6 +1382,8 @@ def cpu_floor(avx2: bool) -> str:
forced = os.environ.get("STRATA_ISA_FLOOR", "").strip().lower()
if forced in ("avx", "none"):
return forced
+ if is_arm64():
+ return ""
if avx2:
return ""
if WIN:
@@ -1457,7 +1468,21 @@ def gpus():
for line in s.strip().splitlines():
try:
idx, name, mem, cc, drv = [x.strip() for x in line.split(",")]
- found.append({"index": int(idx), "name": name, "vram_gb": float(mem) / 1024.0, "arch": cc.replace(".", ""),
+ try:
+ mem_gb = float(mem) / 1024.0 # nvidia-smi nounits: MiB
+ except ValueError:
+ if not is_arm64():
+ raise # as before on x86: a card whose memory nvidia-smi does not report is not listed
+ # GB10 unified memory: memory.total is [N/A]. The pool is the machine RAM.
+ mem_gb = 0.0
+ try:
+ for info in open("/proc/meminfo"):
+ if info.startswith("MemTotal:"):
+ mem_gb = int(info.split()[1]) / 1024.0 / 1024.0
+ break
+ except OSError:
+ pass
+ found.append({"index": int(idx), "name": name, "vram_gb": mem_gb, "arch": cc.replace(".", ""),
"driver": drv})
except ValueError:
continue
@@ -9014,7 +9039,7 @@ def main() -> int:
pf = page_file_gb()
if pf is not None and pf < 4:
warn(page_file_advice(pf, page_file_setting()))
- ok(f"CPU: {cpu} ({'AVX-512' if avx512 else 'AVX2' if avx2 else 'no AVX2'})")
+ ok(f"CPU: {cpu} ({'ARM64' if is_arm64() else 'AVX-512' if avx512 else 'AVX2' if avx2 else 'no AVX2'})")
link = None if hip else pcie_link(int(gpu.get("index", 0)))
if link is not None:
line, problem = pcie_lines(link)
@@ -9024,7 +9049,13 @@ def main() -> int:
floor = cpu_floor(avx2)
if floor == "unsupported":
fail("this CPU has neither AVX2 nor SSE4.2; Strata needs at least SSE4.2 (Intel Nehalem, 2008, or newer)")
- if not avx2:
+ if is_arm64():
+ # DGX Spark / GB10 (#3622): no ready-made ARM64 engine; setup compiles it here. A warning, not a stop.
+ warn("ARM64 CPU: setup compiles the engine on this PC (once). The CPU's share of the experts runs on ggml-cpu's "
+ "NEON kernels. Tested: GGUF packs with i-quant experts (IQ3_S). The Q2_0 and Gyro packs need the x86 "
+ "kernels and do not run on this CPU. This port is new and has not been measured against an x86 PC")
+ a.build = True
+ elif not avx2:
# #394 #595 #623: the ready-made engine is AVX2; an older CPU gets one compiled here, whose CPU experts run on
# ggml-cpu's kernels for this CPU. Experimental: measured only on newer CPUs with the older path forced, and by
# users on a few Xeons. A warning, not a stop.
diff --git a/src/core/expert_source.cpp b/src/core/expert_source.cpp
index b8ad8fb42..43ec2615a 100644
--- a/src/core/expert_source.cpp
+++ b/src/core/expert_source.cpp
@@ -3184,6 +3184,10 @@ void RouterLookahead::run() {
// one (kq_avx1.cpp) or, without AVX, the same sums in plain C++. From the Strata_Dirigo fork (rwkeyes): an
// AVX-only Xeon E5-2687W died here (vpmovzxwd) on its first request. An estimate only (which experts to
// prefetch); the order of the additions differs between the three, the output does not depend on it.
+#if defined(__aarch64__) && !defined(STRATA_GPU_METAL) // Metal (Apple silicon) keeps the plain loop below: its bf16_rows_dot_multi is a stub
+ strata::kernels::cpu::bf16_rows_dot_multi(routers_[(size_t) layer].data(), (int) n_expert_, (int) n_embd_,
+ x_.data(), (int) nt, logits.data());
+#else
if (strata::kernels::cpu::cpu_avx2_ok()) {
strata::kernels::cpu::bf16_rows_dot_multi(routers_[(size_t) layer].data(), (int) n_expert_, (int) n_embd_,
x_.data(), (int) nt, logits.data());
@@ -3209,6 +3213,7 @@ void RouterLookahead::run() {
}
}
}
+#endif
const int kmax = stats_on_ ? (int) std::min<int64_t>(k_ + 10, n_expert_) : k_;
const int ks[3] = {k_, (int) std::min<int64_t>(k_ + 4, n_expert_), kmax};
std::vector<uint64_t> sm;
diff --git a/src/core/graph.cpp b/src/core/graph.cpp
index 2e9c7e70e..da2c50329 100644
--- a/src/core/graph.cpp
+++ b/src/core/graph.cpp
@@ -1,7 +1,7 @@
// src/core/graph.cpp - P2.S5: the GraphRegistry implementation.
#include "strata/core/graph.hpp"
-#include <immintrin.h>
+#include "strata/platform/cpu_relax.hpp"
#include <chrono>
#include <cstdio>
@@ -97,7 +97,7 @@ bool CapturedGraph::wait_ms(int timeout_ms) const {
if (q == cudaSuccess) return true;
if (q != cudaErrorNotReady) return false; // a real error, not "not finished"
if (std::chrono::steady_clock::now() >= deadline) return false;
- for (int i = 0; i < 64; ++i) _mm_pause();
+ for (int i = 0; i < 64; ++i) strata_cpu_pause();
}
}
diff --git a/src/core/mtp.cpp b/src/core/mtp.cpp
index 292a5d2b4..af59e02f6 100644
--- a/src/core/mtp.cpp
+++ b/src/core/mtp.cpp
@@ -44,11 +44,7 @@
#include <fstream>
#include <sstream>
#include <vector>
-#if defined(_WIN32)
-#include <intrin.h>
-#elif defined(__x86_64__)
-#include <immintrin.h>
-#endif
+#include "strata/platform/cpu_relax.hpp"
#ifdef STRATA_NATIVE_EXPERTS
#include "ggml.h" // --mtp-q4: the head rows' and projections' formats, dequantized and requantized at load
@@ -1641,10 +1637,11 @@ bool MtpDrafter::draft(int T, const int32_t* tokens, int64_t p, int a, int32_t*
uint32_t spins = 0;
while (((volatile int32_t*) h_out_)[j] < 0) {
#if defined(_WIN32) || defined(__x86_64__)
- _mm_pause();
+ strata_cpu_pause();
#endif
if ((++spins & 1023u) == 0 && cudaStreamQuery(cs_) != cudaErrorNotReady) break;
}
+ strata_load_fence(); // the token was seen: its probability is read after it (a no-op on x86)
drafts[j] = ((volatile int32_t*) h_out_)[j];
if (probs) probs[j] = ((volatile float*) h_prob_)[j];
prefetch_ple(drafts[j]);
@@ -1677,10 +1674,11 @@ bool MtpDrafter::draft(int T, const int32_t* tokens, int64_t p, int a, int32_t*
uint32_t spins = 0;
while (((volatile int32_t*) h_out_)[j] < 0) {
#if defined(_WIN32) || defined(__x86_64__)
- _mm_pause();
+ strata_cpu_pause();
#endif
if ((++spins & 1023u) == 0 && cudaStreamQuery(cs_) != cudaErrorNotReady) break;
}
+ strata_load_fence();
};
tl = Clock::now();
wait_step(0);
diff --git a/src/core/session.cpp b/src/core/session.cpp
index 0a25df79d..42a7ff77c 100644
--- a/src/core/session.cpp
+++ b/src/core/session.cpp
@@ -21,14 +21,9 @@
#include <cstring>
#include <vector>
-// `_mm_pause` for the doorbell spin. Guarded because it is x86-only; a target without it still builds, the
-// spin is just less polite to the pipeline.
-#if defined(_MSC_VER) || defined(__x86_64__) || defined(__i386__)
-#include <immintrin.h>
-#define STRATA_SPIN_PAUSE() _mm_pause()
-#else
-#define STRATA_SPIN_PAUSE() ((void) 0)
-#endif
+// The doorbell spin. x86 pauses, aarch64 yields (include/strata/platform/cpu_relax.hpp).
+#include "strata/platform/cpu_relax.hpp"
+#define STRATA_SPIN_PAUSE() strata_cpu_pause()
namespace strata::core {
namespace {
@@ -716,6 +711,7 @@ bool session_loop(const ModelGeometry& g, int64_t pos, int32_t pos_base, Session
if (rang) break; // rung, and the graph is still running: the overlap
STRATA_SPIN_PAUSE();
}
+ strata_load_fence(); // the ring was seen: the GPU's x_f / ids / weights are read after it (a no-op on x86)
if (mid_graph) ++gr.rings_mid_graph;
// the window, from just before the launch to the instant the ring was seen
const auto t_ring = std::chrono::steady_clock::now();
@@ -968,11 +964,12 @@ bool session_run_token(const ModelGeometry& g, int64_t pos, int32_t pos_base, Se
return false;
}
}
+ strata_load_fence(); // as above, for the token loop's wait
const auto t1 = Clock::now();
progress_at("token: the CPU experts of layer", l);
if (pool != nullptr) pool(user, s.db->h_x_f, s.db->h_ids, s.db->h_weights, g.n_embd, s.k, y_miss_host);
std::atomic_thread_fence(std::memory_order_seq_cst);
- _mm_sfence();
+ strata_store_fence();
*flag = want;
const auto t2 = Clock::now();
tg.ms_wait += std::chrono::duration<double, std::milli>(t1 - t0).count();
diff --git a/src/core/verify.cpp b/src/core/verify.cpp
index 5e97b8f52..d5e41255c 100644
--- a/src/core/verify.cpp
+++ b/src/core/verify.cpp
@@ -75,7 +75,7 @@
#include <cstdlib>
#include <cstring>
#include <exception>
-#include <immintrin.h>
+#include "strata/platform/cpu_relax.hpp"
#if defined(STRATA_GPU_METAL)
// The Metal backend's decode window (metal/src/engine/window.cpp): this file's init/run/commit hand over to it.
@@ -517,7 +517,7 @@ bool Verifier::release_gpu_waits(int timeout_ms) {
for (uint32_t* p : {h_flag_, h_flagA_, h_flagB_, hflag_in_h_}) // + an overlapped split's hand-off wait
if (p != nullptr) *(volatile uint32_t*) p = UINT32_MAX;
std::atomic_thread_fence(std::memory_order_seq_cst);
- _mm_sfence();
+ strata_store_fence();
const OnDevice on_device(device_);
const Clock::time_point t0 = Clock::now();
for (cudaStream_t s : {cs_, copy_}) {
@@ -2738,7 +2738,7 @@ void Verifier::stage_inputs(int T, const int32_t* tokens, int64_t pos0) {
if (herr_h_ != nullptr) *(volatile uint32_t*) herr_h_ = 0;
if (hdrop_h_ != nullptr) *(volatile uint32_t*) hdrop_h_ = (g_test_drop > 0 && windows + 1 == g_test_drop) ? 1u : 0u;
std::atomic_thread_fence(std::memory_order_seq_cst);
- _mm_sfence();
+ strata_store_fence();
last_t_ = T;
if (rs_on_ && T >= 1 && T <= 8) ++rs_win_[T];
last_pos0_ = pos0;
@@ -2811,7 +2811,7 @@ bool Verifier::adapt_gate_wait(int32_t layer, std::string& err) {
const Clock::time_point t0 = Clock::now();
uint32_t spins = 0;
while (adapt_gate_->reached() < layer) {
- _mm_pause();
+ strata_cpu_pause();
if ((++spins & 1023u) != 0) continue;
if (Clock::now() - t0 > std::chrono::seconds(20)) {
err = "verify: the adaptive tier's copies for layer " + std::to_string(layer) + " never landed" +
@@ -2917,7 +2917,7 @@ bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool,
const Clock::time_point tp = Clock::now();
if (!ss.ple.table->gather_batch(ple_rows, (size_t) T, h_ple_, err)) return false;
std::atomic_thread_fence(std::memory_order_seq_cst);
- _mm_sfence();
+ strata_store_fence();
ms_host += ms_since(tp);
}
*(volatile uint32_t*) h_flag_ = 1;
@@ -2992,7 +2992,7 @@ bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool,
const Clock::time_point tp = Clock::now();
if (!ss.ple.table->gather_batch(ple_rows, (size_t) T, h_ple_, err)) return false;
std::atomic_thread_fence(std::memory_order_seq_cst);
- _mm_sfence();
+ strata_store_fence();
*flag = 1;
ms_host += ms_since(tp);
} else {
@@ -3008,7 +3008,7 @@ bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool,
uint32_t spins = 0;
progress_at("verify window: waiting for the GPU to reach layer", l);
while (*seq < want) {
- _mm_pause();
+ strata_cpu_pause();
if ((++spins & 1023u) != 0) continue;
const auto now = Clock::now();
if (now - last_flush > std::chrono::microseconds(2000)) {
@@ -3044,6 +3044,7 @@ bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool,
return false;
}
}
+ strata_load_fence(); // the layer rang: the GPU's activations are read after it (a no-op on x86)
const Clock::time_point b = Clock::now();
if (g_trace) trace_ev("RANG", k, l, (int64_t) std::chrono::duration_cast<std::chrono::microseconds>(b - a).count());
VDBG("layer %lld rang\n", (long long) l);
@@ -3064,7 +3065,7 @@ bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool,
(int64_t) ms_since(b)); // aux: ms the CPU experts took
progress_tick();
std::atomic_thread_fence(std::memory_order_seq_cst);
- _mm_sfence();
+ strata_store_fence();
if (h_bal_ != nullptr) { // set_pcie_balance: the step's plan, its CPU and its PCIe experts
const size_t bi = (size_t) ((k / G) * 2 + grp);
bal_ok_[bi] = *(volatile uint32_t*) h_flagA_ == want;
@@ -3090,7 +3091,7 @@ bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool,
const Clock::time_point tp = Clock::now();
if (!ss.ple.table->gather_batch(ple_rows, (size_t) T, h_ple_, err)) return false;
std::atomic_thread_fence(std::memory_order_seq_cst);
- _mm_sfence();
+ strata_store_fence();
ms_host += ms_since(tp);
}
if (!(test_stall && k + 1 == steps)) *flag = want;
@@ -3331,7 +3332,7 @@ void Verifier::fetch_dma(void* ctx, const uint8_t* const* src, int n, size_t byt
void Verifier::publish_plan(void* ctx) {
Verifier* v = (Verifier*) ctx;
- _mm_sfence();
+ strata_store_fence();
*(volatile uint32_t*) v->h_flagA_ = v->cur_layer_ + 1;
}
@@ -3862,7 +3863,7 @@ bool Verifier::run_slot_rows(const int* rows, int S, const int32_t* tokens, cons
if (ar_on()) {
if (ss_->ple.ready() && ple_stage()) {
std::atomic_thread_fence(std::memory_order_seq_cst);
- _mm_sfence();
+ strata_store_fence();
*flag = 1;
}
const cudaError_t se = cudaStreamSynchronize(cs_);
@@ -3886,7 +3887,7 @@ bool Verifier::run_slot_rows(const int* rows, int S, const int32_t* tokens, cons
uint32_t spins = 0;
progress_at("verify batch: waiting for the GPU to reach layer", l);
while (*seq < want) {
- _mm_pause();
+ strata_cpu_pause();
if ((++spins & 1023u) != 0) continue;
const auto now = Clock::now();
if (now - last_flush > std::chrono::microseconds(2000)) {
@@ -3903,6 +3904,7 @@ bool Verifier::run_slot_rows(const int* rows, int S, const int32_t* tokens, cons
return false;
}
}
+ strata_load_fence(); // as above, for the batch loop's wait
const Clock::time_point b = Clock::now();
cur_layer_ = want - 1;
set_plan_slot(0);
@@ -3914,7 +3916,7 @@ bool Verifier::run_slot_rows(const int* rows, int S, const int32_t* tokens, cons
if (remote_opt_) remote_opt_->end();
progress_tick();
std::atomic_thread_fence(std::memory_order_seq_cst);
- _mm_sfence();
+ strata_store_fence();
if (*(volatile uint32_t*) h_flagA_ != want) { // the pool did not publish a plan: an empty one
sink_.counts[0] = 0;
sink_.counts[1] = 0;
@@ -4039,7 +4041,7 @@ int Verifier::batch_serve(PoolMultiFn pool, void* user) {
// run_slot_rows makes). Per-window (ar_on): with a loan in flight the doorbell graph needs the per-layer service.
if (ar_on() && ss_->ple.ready() && ple_stage()) {
std::atomic_thread_fence(std::memory_order_seq_cst);
- _mm_sfence();
+ strata_store_fence();
*h_flag_ = 1;
}
for (;;) {
@@ -4059,7 +4061,7 @@ int Verifier::batch_serve(PoolMultiFn pool, void* user) {
if (remote_opt_) remote_opt_->end();
progress_tick();
std::atomic_thread_fence(std::memory_order_seq_cst);
- _mm_sfence();
+ strata_store_fence();
if (*(volatile uint32_t*) h_flagA_ != want) { // the pool did not publish a plan: an empty one
sink_.counts[0] = 0;
sink_.counts[1] = 0;
@@ -4354,7 +4356,7 @@ int Verifier::service(PoolMultiFn pool, void* user, std::string& err) {
const Clock::time_point tp = Clock::now();
if (!ss.ple.table->gather_batch(pl_ple_rows_.data(), (size_t) T, h_ple_, err)) return false;
std::atomic_thread_fence(std::memory_order_seq_cst);
- _mm_sfence();
+ strata_store_fence();
ms_host += ms_since(tp);
return true;
};
@@ -4405,7 +4407,7 @@ int Verifier::service(PoolMultiFn pool, void* user, std::string& err) {
(int64_t) ms_since(b));
progress_tick();
std::atomic_thread_fence(std::memory_order_seq_cst);
- _mm_sfence();
+ strata_store_fence();
if (*(volatile uint32_t*) h_flagA_ != want) { // the pool did not publish a plan: an empty one
sink_.counts[0] = 0;
sink_.counts[1] = 0;
@@ -4479,7 +4481,7 @@ bool Verifier::pl_commit_async(int n_keep, std::string& err) {
const Clock::time_point t0 = Clock::now();
if (commit_live_) { // its graph reads the words below when it starts: the previous one has (a window ago)
cudaError_t q;
- while ((q = cudaEventQuery(ev_commit_)) == cudaErrorNotReady) _mm_pause();
+ while ((q = cudaEventQuery(ev_commit_)) == cudaErrorNotReady) strata_cpu_pause();
if (q != cudaSuccess) { err = std::string("verify: commit: ") + cudaGetErrorString(q); return false; }
}
h_commit_[0] = n_keep;
diff --git a/src/kernels/cpu/expert_layout.cpp b/src/kernels/cpu/expert_layout.cpp
index 9d296c976..bcbedaa2d 100644
--- a/src/kernels/cpu/expert_layout.cpp
+++ b/src/kernels/cpu/expert_layout.cpp
@@ -10,7 +10,7 @@
#if defined(_MSC_VER)
#include <intrin.h>
#include <immintrin.h>
-#else
+#elif defined(__x86_64__) || defined(__i386__)
#include <cpuid.h>
#endif
#include <fstream>
@@ -39,6 +39,38 @@ int cpu_isa_cap() {
return cap;
}
+#if !defined(__x86_64__) && !defined(__i386__) && !defined(_M_X64) && !defined(_M_IX86)
+bool cpu_avx512_ok() { return false; }
+bool cpu_avx512bw_ok() { return false; }
+bool cpu_avx2_ok() { return false; }
+bool cpu_avx1_ok() { return false; }
+bool cpu_sse42_ok() { return false; }
+bool cpu_gather_fast() { return false; }
+bool cpu_gather_fast_here() { return false; }
+bool cpu_avxvnni_ok() { return false; }
+std::string cpu_name() {
+#if defined(__APPLE__)
+ return "unknown"; // as before: the Metal build's cpuid shim read all zeros
+#else
+ std::ifstream f("/proc/cpuinfo");
+ std::string line;
+ while (std::getline(f, line)) {
+ if (line.rfind("model name", 0) == 0 || line.rfind("Model", 0) == 0) {
+ const size_t c = line.find(':');
+ if (c != std::string::npos) {
+ const size_t b = line.find_first_not_of(" \t", c + 1);
+ if (b != std::string::npos) return line.substr(b);
+ }
+ }
+ }
+#if defined(__aarch64__)
+ return "aarch64";
+#else
+ return "unknown";
+#endif
+#endif // __APPLE__
+}
+#else
bool cpu_avx512_ok() {
static const bool ok = [] {
if (const char* f = std::getenv("STRATA_FORCE_AVX2"); f != nullptr && f[0] == '1') return false;
@@ -186,6 +218,8 @@ bool cpu_sse42_ok() {
return ok;
}
+#endif // x86 cpu probes
+
const char* isa_floor_build() {
#if defined(STRATA_ISA_FLOOR_AVX)
return "avx";
@@ -196,6 +230,7 @@ const char* isa_floor_build() {
#endif
}
+#if defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86)
namespace {
void cpuid_regs(unsigned leaf, unsigned sub, unsigned r[4]) {
#if defined(_MSC_VER)
@@ -207,6 +242,7 @@ void cpuid_regs(unsigned leaf, unsigned sub, unsigned r[4]) {
#endif
}
} // namespace
+#endif // x86 cpuid helper
int iq256_gather_setting() {
static const int s = [] {
@@ -217,6 +253,7 @@ int iq256_gather_setting() {
return s;
}
+#if defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86)
bool cpu_gather_fast() {
static const bool ok = [] {
if (cpu_isa_cap() < 3) return false; // STRATA_FORCE_ISA: as on a CPU that stops at AVX2
@@ -295,6 +332,7 @@ std::string cpu_name() {
const size_t b0 = name.find_first_not_of(' '), b1 = name.find_last_not_of(' ');
return b0 == std::string::npos ? std::string("unknown") : name.substr(b0, b1 - b0 + 1);
}
+#endif // x86 cpuid probes
// ---- the AVX-512 probe and the oracle flag (#795) ----
//
@@ -322,6 +360,9 @@ const char* CpuFeatures::reason() const {
CpuFeatures cpu_features() {
CpuFeatures f;
+#if !defined(__x86_64__) && !defined(__i386__) && !defined(_M_X64) && !defined(_M_IX86)
+ return f; // the AVX-512 kernels are x86-only: no AVX-512 on this architecture
+#else
int reg[4] = {0, 0, 0, 0};
#if defined(_MSC_VER)
__cpuid(reg, 0);
@@ -341,6 +382,7 @@ CpuFeatures cpu_features() {
f.avx512_vnni = (ecx >> 11) & 1u;
f.avx512_vbmi = (ecx >> 1) & 1u;
return f;
+#endif
}
void cpu_require_expert_support() {
diff --git a/src/kernels/cpu/pool.cpp b/src/kernels/cpu/pool.cpp
index cdcc8d792..2d362400d 100644
--- a/src/kernels/cpu/pool.cpp
+++ b/src/kernels/cpu/pool.cpp
@@ -9,7 +9,7 @@
#include <atomic>
#include <cmath>
#include <chrono>
-#include <immintrin.h>
+#include "strata/platform/cpu_relax.hpp"
#if defined(_MSC_VER)
#include <intrin.h> // __cpuidex, __rdtsc (STRATA_POOL_UMWAIT)
#endif
@@ -1010,7 +1010,7 @@ void ExpertPool::worker(int id) {
} else
#endif
{
- _mm_pause();
+ strata_cpu_pause();
if ((++spins & 1023u) != 0) continue;
}
if (std::chrono::steady_clock::now() - parked_at < spin_before_sleep_) continue;
@@ -1131,7 +1131,7 @@ void ExpertPool::wait_parked(const char* what) {
uint32_t spins = 0;
std::chrono::steady_clock::time_point t0{};
while (parked_count() != (uint32_t) n_) {
- _mm_pause();
+ strata_cpu_pause();
if ((++spins & 1023u) != 0) continue;
const auto now = std::chrono::steady_clock::now();
if (spins == 1024u) t0 = now;
@@ -1158,7 +1158,7 @@ void ExpertPool::wait_done(int n) {
else
d = done_.load(std::memory_order_acquire);
if (d >= (uint32_t) n) return; // `>=`: never a wait that an overshoot outlives
- _mm_pause();
+ strata_cpu_pause();
if ((++spins & 1023u) != 0) continue;
const auto now = std::chrono::steady_clock::now();
if (spins == 1024u || d != seen) { t0 = now; seen = d; } // progress restarts the clock
@@ -1408,7 +1408,7 @@ void ExpertPool::run(ExpertJob* jobs, int n) {
// ---- **THE HOST DRAINS TOO (R2.2), INSTEAD OF SPINNING ON `done_`.**
//
- // The loop below used to be `while (done_ != n) _mm_pause();`. The host is pinned to core 0 - the core
+ // The loop below used to be `while (done_ != n) strata_cpu_pause();`. The host is pinned to core 0 - the core
// `physical_cores(true)` deliberately keeps the five workers off - so for the whole drain that core was
// idle while five cores did six cores' worth of work. Measured before the change: 33.7 GB/s against
// 5/6 x 44.14 = 36.8 for five workers and 44.14 for six.
diff --git a/src/kernels/cpu/portable.cpp b/src/kernels/cpu/portable.cpp
new file mode 100644
index 000000000..a8ff501cc
--- /dev/null
+++ b/src/kernels/cpu/portable.cpp
@@ -0,0 +1,214 @@
+// src/kernels/cpu/portable.cpp - the CPU kernel symbols on a non-x86 build (aarch64: DGX Spark / GB10, Grace).
+//
+// Replaces expert.cpp, q2_avx2.cpp, iq_avx2.cpp, iq_avx512.cpp, kq_avx2.cpp and kq_avx1.cpp, which are AVX-only.
+// What a native (GGUF) pack needs from this file is the activation quantizer (expert.cpp's scalar loop, bitwise
+// the same as its AVX-512 version) and the BF16 router dot (NEON). The i-quant / k-quant multi-token kernels
+// report themselves unsupported, so native_expert.cpp takes ggml-cpu's vec_dot, which has NEON paths. The
+// canonical Q2_0 pack (its own blob layout) is refused: those kernels are AVX-512 only. A native pack's Q2_0
+// rows are not routed here: q2_native_kernels() is false when cpu_avx2_ok() is, so they use ggml-cpu too.
+// Local port of the approach in https://github.com/Niko1221/Strata/pull/1077 onto main e8ca9af.
+#include "strata/kernels/cpu/expert.hpp"
+#include "strata/kernels/cpu/iq_avx2.hpp"
+#include "strata/kernels/cpu/iq_avx512.hpp"
+#include "strata/kernels/cpu/kq_avx1.hpp"
+#include "strata/kernels/cpu/kq_avx2.hpp"
+#include "strata/kernels/cpu/iq_avx1.hpp"
+#include "strata/kernels/cpu/nvfp4_avx2.hpp"
+#include "strata/kernels/cpu/nvfp4_avx512.hpp"
+#include "strata/kernels/cpu/tq_cpu.hpp"
+
+#include <atomic>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+
+#if defined(__ARM_NEON)
+#include <arm_neon.h>
+#endif
+
+namespace strata::kernels::cpu {
+namespace {
+
+std::atomic<bool> oracle_q8_0{false};
+
+[[noreturn]] void unsupported(const char* what) {
+ std::fprintf(stderr,
+ "strata: %s is x86-only (AVX2/AVX-512) and this build has no portable version.\n"
+ " On this CPU use a native GGUF pack (IQ2_XS, IQ3_S, IQ4_NL, Q4_K, ...).\n",
+ what);
+ std::abort();
+}
+
+inline float bf16f(uint16_t h) {
+ const uint32_t b = (uint32_t) h << 16;
+ float f;
+ std::memcpy(&f, &b, 4);
+ return f;
+}
+
+} // namespace
+
+
+
+
+
+void act_quant_q8_1(const float* x, int n, ActQ& a) {
+ // The Q8_0 oracle contract (expert.cpp's quantize_oracle_q8_0) is an AVX rounding. The legacy scalar loop
+ // below is what the engine uses; the oracle flag would silently quantize differently here.
+ if (oracle_q8_0.load(std::memory_order_relaxed)) unsupported("the Q8_0 oracle activation quantizer");
+ // expert.cpp's scalar loop: round half away from zero, clamp to +-127.
+ a.nchunks = n / QKA;
+ for (int k = 0; k < a.nchunks; ++k) {
+ const float* xb = x + k * QKA;
+ float amax = 0.f;
+ for (int j = 0; j < QKA; ++j) amax = std::fmax(amax, std::fabs(xb[j]));
+ const float s = amax > 0.f ? amax / 127.f : 0.f;
+ const float inv = s > 0.f ? 1.f / s : 0.f;
+ int32_t sum = 0;
+ int8_t* q = a.q + k * QKA;
+ for (int j = 0; j < QKA; ++j) {
+ const float t = xb[j] * inv;
+ const float r = t + (t >= 0.f ? 0.5f : -0.5f);
+ int v = (int) r;
+ v = v < -127 ? -127 : (v > 127 ? 127 : v);
+ q[j] = (int8_t) v;
+ sum += v;
+ }
+ a.scale[k] = s;
+ a.sum[k] = sum;
+ a.hx[k] = s * (float) sum;
+ }
+}
+
+void act_quant_q8_1_avx2(const float* x, int n, ActQ& a) { act_quant_q8_1(x, n, a); }
+
+void s2_expert_vnni(const uint8_t*, const float*, float*, ExpertScratch&) { unsupported("the Q2_0 expert kernel"); }
+void s2_expert_vnni_q(const uint8_t*, const ActQ&, float*, ExpertScratch&) { unsupported("the Q2_0 expert kernel"); }
+void s2_expert_gu_rows(const uint8_t*, const ActQ&, float*, int, int) { unsupported("the Q2_0 expert kernel"); }
+void s2_expert_down_rows(const uint8_t*, const ActQ&, float*, int, int) { unsupported("the Q2_0 expert kernel"); }
+void s2_expert_gu_rows_multi(const uint8_t*, const ActQ* const*, int, float* const*, int, int) {
+ unsupported("the Q2_0 expert kernel");
+}
+void s2_expert_down_rows_multi(const uint8_t*, const ActQ* const*, int, float* const*, int, int) {
+ unsupported("the Q2_0 expert kernel");
+}
+void s2_expert_vnni_multi(const uint8_t*, const ActQ* const*, int, float* const*, ExpertScratchMulti&) {
+ unsupported("the Q2_0 expert kernel");
+}
+void q2_0_gguf_rows_multi(const uint8_t*, size_t, int, const ActQ* const*, int, float* const*, int, int) {
+ unsupported("the Q2_0 GGUF row kernel");
+}
+void q2_0_gguf_rows_multi_avx2(const uint8_t*, size_t, int, const ActQ* const*, int, float* const*, int, int) {
+ unsupported("the Q2_0 GGUF row kernel");
+}
+bool q2_bitplane_enabled() { return false; }
+void q2_0_gguf_rows_multi_avx2_legacy(const uint8_t*, size_t, int, const ActQ* const*, int, float* const*, int, int) {
+ unsupported("the Q2_0 GGUF row kernel");
+}
+void q2_0_gguf_rows_multi_avx2_v(bool, const uint8_t*, size_t, int, const ActQ* const*, int, float* const*, int, int) {
+ unsupported("the Q2_0 GGUF row kernel");
+}
+void s2_expert_scalar(const uint8_t*, const float*, float*, bool) { unsupported("the Q2_0 scalar oracle"); }
+
+void iq512_gu_rows(int, const uint8_t*, size_t, size_t, int, const void* const*, int, float* const*, int, int) {
+ unsupported("the AVX-512 i-quant kernel");
+}
+void iq512_rows(int, const uint8_t*, size_t, int, const void* const*, int, float* const*, int, int) {
+ unsupported("the AVX-512 i-quant kernel");
+}
+bool iq512_gather_on() noexcept { return false; }
+bool iq512x_supported(int) noexcept { return false; }
+void iq512x_rows(int, const uint8_t*, size_t, int, const void* const*, int, float* const*, int, int) {
+ unsupported("the AVX-512 exact i-quant kernel");
+}
+int iq256_variant() noexcept { return 0; }
+int iq256_variants() noexcept { return 0; }
+void iq256_gu_rows_v(int, int, const uint8_t*, size_t, size_t, int, const void* const*, int, float* const*, int, int) {
+ unsupported("the AVX-2 i-quant kernel");
+}
+void iq256_rows_v(int, int, const uint8_t*, size_t, int, const void* const*, int, float* const*, int, int) {
+ unsupported("the AVX-2 i-quant kernel");
+}
+void iq256_gu_rows(int, const uint8_t*, size_t, size_t, int, const void* const*, int, float* const*, int, int) {
+ unsupported("the AVX-2 i-quant kernel");
+}
+void iq256_rows(int, const uint8_t*, size_t, int, const void* const*, int, float* const*, int, int) {
+ unsupported("the AVX-2 i-quant kernel");
+}
+void iq4nl256_down_rows_v(int, const uint8_t*, size_t, int, const void* const*, int, float* const*, int, int) {
+ unsupported("the AVX-2 IQ4_NL kernel");
+}
+void iq4nl256_down_rows(const uint8_t*, size_t, int, const void* const*, int, float* const*, int, int) {
+ unsupported("the AVX-2 IQ4_NL kernel");
+}
+void q8k_quant_avx2(const float*, void*, int64_t) { unsupported("the AVX-2 Q8_K quantizer"); }
+void kq256_gu_rows(int, const uint8_t*, size_t, size_t, int, const void* const*, int, float* const*, int, int) {
+ unsupported("the AVX-2 k-quant kernel");
+}
+void kq256_rows(int, const uint8_t*, size_t, int, const void* const*, int, float* const*, int, int) {
+ unsupported("the AVX-2 k-quant kernel");
+}
+void bf16_rows_dot_multi_avx1(const uint16_t*, int, int, const float*, int, float*) {
+ unsupported("the AVX1 router dot");
+}
+
+void bf16_rows_dot(const uint16_t* w, int rows, int cols, const float* x, float* out) {
+ bf16_rows_dot_multi(w, rows, cols, x, 1, out);
+}
+
+void bf16_rows_dot_multi(const uint16_t* w, int rows, int cols, const float* x, int nt, float* out) {
+ for (int r = 0; r < rows; ++r) {
+ const uint16_t* wr = w + (size_t) r * (size_t) cols;
+ for (int t = 0; t < nt; ++t) {
+ const float* xt = x + (size_t) t * cols;
+ int c = 0;
+ float s = 0.f;
+#if defined(__ARM_NEON)
+ float32x4_t acc0 = vdupq_n_f32(0.f), acc1 = vdupq_n_f32(0.f);
+ for (; c + 8 <= cols; c += 8) {
+ const uint16x8_t h = vld1q_u16(wr + c);
+ const float32x4_t w0 = vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(h), 16));
+ const float32x4_t w1 = vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(h), 16));
+ acc0 = vfmaq_f32(acc0, w0, vld1q_f32(xt + c));
+ acc1 = vfmaq_f32(acc1, w1, vld1q_f32(xt + c + 4));
+ }
+ s = vaddvq_f32(vaddq_f32(acc0, acc1));
+#endif
+ for (; c < cols; ++c) s += bf16f(wr[c]) * xt[c];
+ out[(size_t) t * rows + r] = s;
+ }
+ }
+}
+
+
+// aarch64 has none of the x86 kernels: these names are referenced by shared code but reached only behind
+// the x86 feature checks (all of them report unsupported here).
+[[noreturn]] static void cpu_kernel_unreachable(const char* name) {
+ std::fprintf(stderr, "strata: %s is an x86 kernel and cannot run on this CPU\n", name);
+ std::abort();
+}
+void act_quant_q8_1_avx1(const float* x, int n, ActQ& a) { cpu_kernel_unreachable("act_quant_q8_1_avx1"); }
+void iq128_down_rows(int ggml_type, const uint8_t* w, size_t row_bytes, int n, const void* const* hq, int nt, float* const* out, int r0, int r1) { cpu_kernel_unreachable("iq128_down_rows"); }
+bool iq128_down_supported(int ggml_type) noexcept { return false; }
+void iq128_gu_rows(int ggml_type, const uint8_t* blob, size_t gu_row, size_t up_off, int n, const void* const* act, int nt, float* const* ff, int r0, int r1) { cpu_kernel_unreachable("iq128_gu_rows"); }
+bool iq128_supported(int ggml_type) noexcept { return false; }
+void iq256_gu_rows_exact_one(int ggml_type, const uint8_t* blob, size_t gu_row, size_t up_off, int n, const void* act, float* ff, int r0, int r1) { cpu_kernel_unreachable("iq256_gu_rows_exact_one"); }
+bool nvfp4_256_fits(int n) { return false; }
+void nvfp4_256_gu_rows(const uint8_t* blob, size_t gu_row, size_t up_off, int n, const void* const* act, int nt, float* const* ff, int r0, int r1, float s_gate, float s_up) { cpu_kernel_unreachable("nvfp4_256_gu_rows"); }
+void nvfp4_256_rows(const uint8_t* w, size_t row_bytes, int n, const void* const* act, int nt, float* const* out, int r0, int r1, float scale) { cpu_kernel_unreachable("nvfp4_256_rows"); }
+bool nvfp4_512_fits(int n) { return false; }
+void nvfp4_512_gu_rows(const uint8_t* blob, size_t gu_row, size_t up_off, int n, const void* const* act, int nt, float* const* ff, int r0, int r1, float s_gate, float s_up) { cpu_kernel_unreachable("nvfp4_512_gu_rows"); }
+void nvfp4_512_rows(const uint8_t* w, size_t row_bytes, int n, const void* const* act, int nt, float* const* out, int r0, int r1, float scale) { cpu_kernel_unreachable("nvfp4_512_rows"); }
+void q2_0_gguf_rows_multi_avx1_legacy(const uint8_t* w, size_t row_bytes, int nblocks, const ActQ* const* a, int nt, float* const* out, int r0, int r1) { cpu_kernel_unreachable("q2_0_gguf_rows_multi_avx1_legacy"); }
+void q2_0_gguf_rows_multi_avx1(const uint8_t* w, size_t row_bytes, int nblocks, const ActQ* const* a, int nt, float* const* out, int r0, int r1) { cpu_kernel_unreachable("q2_0_gguf_rows_multi_avx1"); }
+void q8k_quant_avx1(const float* x, void* y, int64_t n) { cpu_kernel_unreachable("q8k_quant_avx1"); }
+void tq_gu_rows(TqIsa isa, int ggml_type, const uint8_t* blob, size_t gu_row, size_t up_off, int n, const void* const* act, int nt, float* const* ff, int r0, int r1) { cpu_kernel_unreachable("tq_gu_rows"); }
+TqIsa tq_isa() { return TqIsa::scalar; }
+int tq_kbits(int ggml_type) noexcept { return 0; }
+bool tq_kernels_enabled() { return false; }
+void tq_rows(TqIsa isa, int ggml_type, const uint8_t* w, size_t row_bytes, int n, const void* const* act, int nt, float* const* out, int r0, int r1) { cpu_kernel_unreachable("tq_rows"); }
+
+bool tq_supported(int) noexcept { return false; }
+
+} // namespace strata::kernels::cpu
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index 75d404ac9..63ae3e34b 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -3447,6 +3447,7 @@ int main(int argc, char** argv) {
// compiled `/arch:AVX512`, so on a CPU without those features it does not fail - it executes an illegal
// instruction at some unpredictable token. Refusing at second zero is the whole point of P2.S3's check.
strata::kernels::cpu::expert_set_oracle_q8_0(o.cpu_oracle_q8_0);
+#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) || defined(STRATA_GPU_METAL)
// ... and nothing runs on a CPU without AVX2: every CPU expert kernel is AVX2 at least (the AVX-512 ones are
// chosen above it), and so is ggml-cpu in the release build, which the native pack's layout load initializes
// next. Refused here, by name, rather than an illegal instruction in the first expert.
@@ -3484,6 +3485,14 @@ int main(int argc, char** argv) {
"and the normal build is faster on it\n", isa_floor);
}
+#else
+ if (o.cpu_oracle_q8_0) {
+ std::fprintf(stderr, "strata generate: --cpu-oracle-q8-0 is the x86 Q8_0 activation contract; this CPU uses the "
+ "scalar quantizer\n");
+ return 2;
+ }
+#endif
+
std::string err;
if (!o.native_head_gguf.empty() && !o.stream_token) {
std::fprintf(stderr, "--native-head-gguf requires --stream-token\n");
diff --git a/tests/terminal/test_setup_arm64.py b/tests/terminal/test_setup_arm64.py
new file mode 100644
index 000000000..7f51056a5
--- /dev/null
+++ b/tests/terminal/test_setup_arm64.py
@@ -0,0 +1,68 @@
+"""setup.py on an ARM64 machine (DGX Spark / GB10, #3622): the GPU line, the CPU probe and the older-CPU floor.
+x86 machines must keep their old answers: a card whose memory nvidia-smi does not report stays unlisted there."""
+
+from __future__ import annotations
+
+import os
+import sys
+import unittest
+from pathlib import Path
+from unittest import mock
+
+ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(ROOT))
+
+import setup # noqa: E402
+
+GB10 = "0, NVIDIA GB10, [N/A], 12.1, 580.95.05\n"
+MEMINFO = "MemTotal: 130000000 kB\nMemFree: 1000000 kB\n"
+ARM_CPUINFO = ("processor\t: 0\nBogoMIPS\t: 2000.00\nFeatures\t: fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics\n"
+ "CPU implementer\t: 0x41\n")
+
+
+class UnifiedMemoryCard(unittest.TestCase):
+ def test_a_card_without_a_memory_figure_is_unlisted_on_x86_as_before(self):
+ with mock.patch.object(setup, "out", return_value=GB10), mock.patch.object(setup, "is_arm64", return_value=False):
+ self.assertEqual(setup.gpus(), [])
+
+ def test_gb10_takes_the_pool_size_from_the_machine_ram(self):
+ with mock.patch.object(setup, "out", return_value=GB10), mock.patch.object(setup, "is_arm64", return_value=True), \
+ mock.patch("builtins.open", mock.mock_open(read_data=MEMINFO)):
+ found = setup.gpus()
+ self.assertEqual(len(found), 1)
+ self.assertEqual(found[0]["name"], "NVIDIA GB10")
+ self.assertEqual(found[0]["arch"], "121")
+ self.assertAlmostEqual(found[0]["vram_gb"], 130000000 / 1024.0 / 1024.0, places=3)
+
+ def test_a_normal_card_is_unchanged(self):
+ with mock.patch.object(setup, "out", return_value="0, NVIDIA GeForce RTX 5070, 12227, 12.0, 580.1\n"):
+ found = setup.gpus()
+ self.assertAlmostEqual(found[0]["vram_gb"], 12227 / 1024.0, places=6)
+
+
+class ArmCpu(unittest.TestCase):
+ def test_an_arm_cpuinfo_without_the_x86_flags_line_does_not_crash(self):
+ with mock.patch.object(setup, "WIN", False), mock.patch("builtins.open", mock.mock_open(read_data=ARM_CPUINFO)):
+ name, avx2, avx512 = setup.cpu_info()
+ self.assertFalse(avx2)
+ self.assertFalse(avx512)
+
+ def test_an_x86_cpuinfo_is_read_as_before(self):
+ text = "flags\t\t: fpu avx avx2 avx512f avx512bw avx512vl avx512_vnni avx512vbmi\nmodel name\t: Test CPU\n"
+ with mock.patch.object(setup, "WIN", False), mock.patch("builtins.open", mock.mock_open(read_data=text)):
+ name, avx2, avx512 = setup.cpu_info()
+ self.assertEqual((name, avx2, avx512), ("Test CPU", True, True))
+
+ def test_no_older_cpu_floor_on_arm(self):
+ with mock.patch.dict(os.environ, {"STRATA_ISA_FLOOR": ""}), mock.patch.object(setup, "is_arm64", return_value=True):
+ self.assertEqual(setup.cpu_floor(False), "")
+
+ def test_the_floor_on_x86_without_avx2_is_still_asked_for(self):
+ with mock.patch.dict(os.environ, {"STRATA_ISA_FLOOR": ""}), mock.patch.object(setup, "is_arm64", return_value=False), \
+ mock.patch.object(setup, "WIN", False), \
+ mock.patch("builtins.open", mock.mock_open(read_data="flags\t: fpu sse4_2 popcnt avx\n")):
+ self.assertEqual(setup.cpu_floor(False), "avx")
+
+
+if __name__ == "__main__":
+ unittest.main()