Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Engine for the DGX Spark: the arm64 CUDA build on top of #3622

Niko1221f262b81b2897
diff --git a/.gitignore b/.gitignore
index 88ad4c7b5..65457fdc0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -89,3 +89,9 @@ node_modules/
# screenshots and checkpoints from the Mac rebuild live outside git (StrataSplit\evidence-archive)
/apps/host/docs/evidence/
/apps/host/docs/checkpoints/
+
+# DGX Spark build outputs and local venv (docs/DGX_SPARK_LINUX.md)
+/out-spark/
+/build-spark/
+/build-vision-spark/
+/.venv-build/
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 369d699c0..2bb8f200c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -131,6 +131,11 @@ set(_strata_tests_default OFF)
if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tests/CMakeLists.txt AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/bench/micro)
set(_strata_tests_default ON)
endif()
+if(NOT STRATA_CPU_X86)
+ # arm64 Linux (DGX Spark): much of the suite drives the x86 CPU kernels, which this build does not have. A default, not a
+ # prohibition: -DSTRATA_BUILD_TESTS=ON still builds what builds (classifying the tests by capability is a follow-up).
+ set(_strata_tests_default OFF)
+endif()
option(STRATA_BUILD_TESTS "Build the test suite" ${_strata_tests_default})
option(STRATA_BUILD_CONVERSATION_TESTS "Build standalone conversation cache tests" OFF)
option(STRATA_WERROR "Treat compiler warnings as errors" OFF)
@@ -1844,7 +1849,14 @@ if(STRATA_NATIVE_EXPERTS)
set(GGML_BACKEND_DL OFF CACHE BOOL "" FORCE)
set(GGML_BUILD_TESTS OFF CACHE BOOL "" FORCE)
set(GGML_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
- if(STRATA_PORTABLE)
+ if(STRATA_PORTABLE AND NOT STRATA_CPU_X86)
+ # arm64 (DGX Spark: Cortex-X925 + A725): ggml-cpu for any Armv8.2 CPU with dot product, FP16 and int8 matmul instead of the build
+ # host's, so the engine built here does not depend on which core the compiler ran on. Without it GGML_NATIVE OFF would leave
+ # ggml-cpu at plain armv8-a, and a native pack's CPU experts would lose the NEON dot products.
+ set(GGML_NATIVE OFF CACHE BOOL "" FORCE)
+ set(STRATA_ARM64_ARCH "armv8.2-a+dotprod+fp16+i8mm" CACHE STRING "arm64: ggml-cpu's -march for a STRATA_PORTABLE build")
+ set(GGML_CPU_ARM_ARCH "${STRATA_ARM64_ARCH}" CACHE STRING "" FORCE)
+ elseif(STRATA_PORTABLE)
set(GGML_NATIVE OFF CACHE BOOL "" FORCE)
foreach(_f GGML_AVX GGML_AVX2 GGML_FMA GGML_F16C GGML_BMI2)
set(${_f} ON CACHE BOOL "" FORCE)
@@ -1897,6 +1909,10 @@ if(STRATA_NATIVE_EXPERTS)
src/kernels/cpu/kq_avx2.cpp src/kernels/cpu/kq_avx1.cpp src/kernels/cpu/q2_avx1.cpp
src/kernels/cpu/iq_avx1.cpp src/kernels/cpu/nvfp4_avx512.cpp src/kernels/cpu/nvfp4_avx2.cpp
src/kernels/cpu/tq_cpu.cpp src/kernels/cpu/tq_avx2.cpp src/kernels/cpu/tq_avx512.cpp)
+ else()
+ # the trellis dispatcher alone (the DeepSeek engine's CPU lane and Gyro): tq_cpu.cpp runs its scalar rows when the x86 units are
+ # not compiled (the Metal build does the same)
+ target_sources(strata_kernels_cpu PRIVATE src/kernels/cpu/tq_cpu.cpp)
endif()
target_include_directories(strata_kernels_cpu PRIVATE third_party/ggml)
if(MSVC)
@@ -1949,6 +1965,8 @@ if(STRATA_NATIVE_EXPERTS)
target_link_libraries(dequant_q5_1_test PRIVATE strata_artifact ggml-base)
add_test(NAME dequant_q5_1_test COMMAND dequant_q5_1_test)
endif()
+ # the x86 kernels' parity programs and benches (to the matching endif below): not on arm64, which has none of those kernels
+ if(STRATA_CPU_X86)
# the router lookahead's dot on each rung this CPU has (AVX2, the older CPUs' AVX1) against a double reference
add_executable(router_dot_parity src/kernels/cpu/router_dot_parity.cpp)
target_link_libraries(router_dot_parity PRIVATE strata_kernels_cpu)
@@ -2009,6 +2027,7 @@ if(STRATA_NATIVE_EXPERTS)
add_test(NAME q8k_quant_parity COMMAND q8k_quant_parity)
add_test(NAME q8k_quant_parity_avx COMMAND ${CMAKE_COMMAND} -E env STRATA_FORCE_ISA=avx $<TARGET_FILE:q8k_quant_parity>)
endif()
+ endif() # STRATA_CPU_X86
set(_strata_mmq_types q2_0 iq2_xxs iq2_xs iq2_s iq3_xxs iq3_s iq4_nl iq4_xs q8_0)
# APR: the agention llama.cpp fork has MMQ instances for its trellis expert types (f62fdead0); any other ggml
# tree leaves them to the FP16 prompt path
diff --git a/spark/build_engine.sh b/spark/build_engine.sh
new file mode 100755
index 000000000..0b9a7f736
--- /dev/null
+++ b/spark/build_engine.sh
@@ -0,0 +1,94 @@
+#!/bin/sh
+# Build the Strata CUDA engine for an NVIDIA DGX Spark (GB10, sm_121, arm64 Linux) into a ready-made engine folder.
+#
+# Runs on arm64 Linux with a CUDA 13 toolkit (13.0 is the release toolkit), on the Spark itself.
+#
+# spark/build_engine.sh # engine + DeepSeek engine + picture reader -> ./out-spark/engine
+# BUILD_VISION=0 spark/build_engine.sh # text only (faster)
+# BUILD_DSV41=0 spark/build_engine.sh # without the DeepSeek-V4.1 engine (strata-dsv41)
+# ARCHS="121" OUT=/tmp/eng JOBS=8 spark/build_engine.sh
+#
+# The result is the folder setup.py keeps a ready-made engine in: copy it to Strata/engine on the Spark and run
+# ./setup.sh there (docs/DGX_SPARK_LINUX.md). packaging/build_cuda_linux_aarch64.sh runs this and then makes the engine
+# package for the signed feed. The flags are packaging/build_cuda_linux.sh's (the release build), for arm64:
+# STRATA_PORTABLE makes ggml-cpu a baseline Armv8.2 build (dotprod + fp16 + i8mm, what a Grace CPU has) instead of the build
+# host's, and both programs find the CUDA libraries beside themselves ($ORIGIN), never the toolkit they were built with.
+set -eu
+repo=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd -P)
+archs=${ARCHS:-121}
+out=${OUT:-$repo/out-spark/engine}
+build=${BUILD:-$repo/build-spark}
+vision_dir=${BUILD_VISION_DIR:-$repo/build-vision-spark}
+jobs=${JOBS:-$(nproc)}
+py=${PYTHON:-python3}
+cmake=${CMAKE:-cmake}
+llama=${STRATA_GGML_DIR:-$repo/third_party/llama.cpp}
+
+case "$(uname -s)-$(uname -m)" in
+ Linux-aarch64|Linux-arm64) ;;
+ *) echo "build_engine.sh: this builds on arm64 Linux (the DGX Spark)" >&2; exit 2 ;;
+esac
+# DGX OS keeps the toolkit outside PATH for a plain user; the release build's toolkit is 13.0
+if ! command -v nvcc >/dev/null 2>&1 && [ -x /usr/local/cuda/bin/nvcc ]; then
+ PATH=/usr/local/cuda/bin:$PATH
+ export PATH
+fi
+cudafull=$(nvcc --version | sed -n 's/.*release \([0-9][0-9]*\.[0-9][0-9]*\).*/\1/p' | head -1)
+case "$cudafull" in
+ 13.0|13.1) ;;
+ 13.2) echo "build_engine.sh: warning: CUDA 13.2.0 / 13.2.1 miscompiled the engine for sm_120 (13.2.2 is fixed;" \
+ "docs/TROUBLESHOOTING.md); 13.0 is the release toolkit" >&2 ;;
+ 13.*) echo "build_engine.sh: note: CUDA $cudafull has not been tried for this build; 13.0 is the release toolkit" >&2 ;;
+ *) echo "build_engine.sh: needs the CUDA 13 toolkit (nvcc here: ${cudafull:-missing})" >&2; exit 2 ;;
+esac
+if [ ! -f "$llama/ggml/CMakeLists.txt" ]; then
+ # llama.cpp at the pinned commit (ggml for the engine, mtmd for pictures), exactly as setup.py fetches it
+ (cd "$repo" && "$py" -c "import setup; print(setup.get_llama_cpp())")
+ [ -f "$llama/ggml/CMakeLists.txt" ] || { echo "build_engine.sh: no llama.cpp tree at $llama after fetching it (STRATA_GGML_DIR=<tree> names another)" >&2; exit 2; }
+fi
+# the commit the DeepSeek engine prints with --version: a checkout gives it to CMake itself; a tree copy without .git needs COMMIT=<hex>
+want_dsv41=OFF; [ "${BUILD_DSV41:-1}" = 1 ] && want_dsv41=ON
+if [ "$want_dsv41" = ON ] && [ ! -e "$repo/.git" ] && [ ! -f "$repo/BUILD_ID" ]; then
+ [ -n "${COMMIT:-}" ] || { echo "build_engine.sh: $repo has no .git and no BUILD_ID: set COMMIT=<hex> so the DeepSeek engine can print its commit" >&2; exit 2; }
+ printf '%s\n' "$(printf '%s' "$COMMIT" | cut -c1-9)" > "$repo/BUILD_ID"
+fi
+
+real=""; for a in $(echo "$archs" | tr ';,' ' '); do real="$real;${a}-real"; done; real=${real#;}
+last=$(echo "$archs" | tr ';,' ' ' | awk '{print $NF}')
+cd "$repo"
+"$cmake" -G Ninja -S . -B "$build" -DCMAKE_BUILD_TYPE=Release -DSTRATA_ENABLE_CUDA=ON -DSTRATA_BUILD_TESTS=OFF \
+ -DSTRATA_NATIVE_EXPERTS=ON -DSTRATA_PORTABLE=ON -DSTRATA_Q6K_EXPERTS=ON -DSTRATA_MMQ_KQUANTS=ON \
+ -DSTRATA_BUILD_DSV41="$want_dsv41" -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON -DCMAKE_SKIP_BUILD_RPATH=OFF '-DCMAKE_INSTALL_RPATH=$ORIGIN' \
+ "-DCMAKE_CUDA_ARCHITECTURES=${real};${last}-virtual" -DSTRATA_GGML_DIR="$llama"
+rm -f "$build/strata-dsv41"
+if [ "$want_dsv41" = ON ]; then
+ "$cmake" --build "$build" --target strata strata_kq strata-dsv41 -j "$jobs"
+else
+ "$cmake" --build "$build" --target strata strata_kq -j "$jobs"
+fi
+
+if [ "${BUILD_VISION:-1}" = 1 ]; then
+ "$cmake" -G Ninja -S tools/vision -B "$vision_dir" -DCMAKE_BUILD_TYPE=Release -DLLAMA_DIR="$llama" \
+ -DSTRATA_VISION_CUDA=ON -DSTRATA_PORTABLE=ON -DGGML_CUDA_NCCL=OFF -DGGML_CPU_ARM_ARCH=armv8.2-a+dotprod+fp16+i8mm \
+ "-DCMAKE_CUDA_ARCHITECTURES=${real};${last}-virtual"
+ "$cmake" --build "$vision_dir" --target strata-vision -j "$jobs"
+fi
+
+ver=$(sed -n 's/^[[:space:]]*project([^)]*VERSION[[:space:]]*\([0-9][0-9.]*\).*/\1/p' CMakeLists.txt | head -1)
+rm -rf "$out"
+mkdir -p "$out"
+cp "$build/strata" "$out/"
+[ -f "$build/libstrata_kq.so" ] && cp "$build/libstrata_kq.so" "$out/"
+# DeepSeek-V4.1's engine beside it: setup.py takes engine/strata-dsv41 (dsv41_exe)
+[ "$want_dsv41" = ON ] && [ -f "$build/strata-dsv41" ] && cp "$build/strata-dsv41" "$out/"
+vision_json=""
+if [ "${BUILD_VISION:-1}" = 1 ] && [ -f "$vision_dir/bin/strata-vision" ]; then
+ cp "$vision_dir/bin/strata-vision" "$out/"
+ vision_json=", \"vision_archs\": \"$archs\""
+fi
+# the CUDA libraries the engine loads (cuBLAS, cuBLASLt, cudart) go beside it: the Spark needs only its driver
+"$py" packaging/collect_libs.py --binary "$build/strata" --search /usr/local/cuda/lib64 --out "$out" --also 'libcublasLt.so*'
+echo "{\"version\": \"$ver\", \"source\": \"spark/build_engine.sh\", \"platform\": \"linux-aarch64\", \"archs\": \"$archs\", \"ptx\": true, \"cuda\": \"$cudafull\", \"portable\": true$vision_json}" > "$out/BUILD.json"
+"$out/strata" --version 2>/dev/null || true
+[ "$want_dsv41" = ON ] && { "$out/strata-dsv41" --version 2>/dev/null || true; }
+echo "built: $out"
diff --git a/src/dsv41/ds4_tq_cpu.cpp b/src/dsv41/ds4_tq_cpu.cpp
index afafb584a..7aefb6c82 100644
--- a/src/dsv41/ds4_tq_cpu.cpp
+++ b/src/dsv41/ds4_tq_cpu.cpp
@@ -53,8 +53,10 @@ void dots(int type, const uint8_t* w, size_t rb, int n, const void* const* act,
const int kb = kc::tq_kbits(type);
const int16_t* tab = kc::tq_state_table_for(type);
switch (kc::tq_isa()) {
+#if defined(__x86_64__) || defined(_M_X64) // the AVX units exist on x86-64 builds only (arm64: the scalar rows)
case kc::TqIsa::avx512: kc::tq_detail::tq_rows_avx512(kb, tab, w, rb, n, ap, nt, out, r0, r1); break;
case kc::TqIsa::avx2: kc::tq_detail::tq_rows_avx2(kb, tab, w, rb, n, ap, nt, out, r0, r1); break;
+#endif
default: kc::tq_detail::tq_rows_scalar(kb, tab, w, rb, n, ap, nt, out, r0, r1); break;
}
}
diff --git a/src/kernels/cpu/portable.cpp b/src/kernels/cpu/portable.cpp
index a8ff501cc..09cc3271a 100644
--- a/src/kernels/cpu/portable.cpp
+++ b/src/kernels/cpu/portable.cpp
@@ -15,7 +15,6 @@
#include "strata/kernels/cpu/iq_avx1.hpp"
#include "strata/kernels/cpu/nvfp4_avx2.hpp"
#include "strata/kernels/cpu/nvfp4_avx512.hpp"
-#include "strata/kernels/cpu/tq_cpu.hpp"
#include <atomic>
#include <cmath>
@@ -203,12 +202,6 @@ void nvfp4_512_rows(const uint8_t* w, size_t row_bytes, int n, const void* const
void q2_0_gguf_rows_multi_avx1_legacy(const uint8_t* w, size_t row_bytes, int nblocks, const ActQ* const* a, int nt, float* const* out, int r0, int r1) { cpu_kernel_unreachable("q2_0_gguf_rows_multi_avx1_legacy"); }
void q2_0_gguf_rows_multi_avx1(const uint8_t* w, size_t row_bytes, int nblocks, const ActQ* const* a, int nt, float* const* out, int r0, int r1) { cpu_kernel_unreachable("q2_0_gguf_rows_multi_avx1"); }
void q8k_quant_avx1(const float* x, void* y, int64_t n) { cpu_kernel_unreachable("q8k_quant_avx1"); }
-void tq_gu_rows(TqIsa isa, int ggml_type, const uint8_t* blob, size_t gu_row, size_t up_off, int n, const void* const* act, int nt, float* const* ff, int r0, int r1) { cpu_kernel_unreachable("tq_gu_rows"); }
-TqIsa tq_isa() { return TqIsa::scalar; }
-int tq_kbits(int ggml_type) noexcept { return 0; }
-bool tq_kernels_enabled() { return false; }
-void tq_rows(TqIsa isa, int ggml_type, const uint8_t* w, size_t row_bytes, int n, const void* const* act, int nt, float* const* out, int r0, int r1) { cpu_kernel_unreachable("tq_rows"); }
-
-bool tq_supported(int) noexcept { return false; }
+// (the trellis rows are tq_cpu.cpp's: its scalar kernel runs here, as in the Metal build)
} // namespace strata::kernels::cpu