Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Engine for the DGX Spark: the arm64 CUDA build on top of #3622
Niko1221f262b81b2897
diff --git a/.gitignore b/.gitignoreindex 88ad4c7b5..65457fdc0 100644--- a/.gitignore+++ b/.gitignore@@ -89,3 +89,9 @@ node_modules/# screenshots and checkpoints from the Mac rebuild live outside git (StrataSplit\evidence-archive)/apps/host/docs/evidence//apps/host/docs/checkpoints/++# DGX Spark build outputs and local venv (docs/DGX_SPARK_LINUX.md)+/out-spark/+/build-spark/+/build-vision-spark/+/.venv-build/diff --git a/CMakeLists.txt b/CMakeLists.txtindex 369d699c0..2bb8f200c 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -131,6 +131,11 @@ set(_strata_tests_default OFF)if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tests/CMakeLists.txt AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/bench/micro)set(_strata_tests_default ON)endif()+if(NOT STRATA_CPU_X86)+ # arm64 Linux (DGX Spark): much of the suite drives the x86 CPU kernels, which this build does not have. A default, not a+ # prohibition: -DSTRATA_BUILD_TESTS=ON still builds what builds (classifying the tests by capability is a follow-up).+ set(_strata_tests_default OFF)+endif()option(STRATA_BUILD_TESTS "Build the test suite" ${_strata_tests_default})option(STRATA_BUILD_CONVERSATION_TESTS "Build standalone conversation cache tests" OFF)option(STRATA_WERROR "Treat compiler warnings as errors" OFF)@@ -1844,7 +1849,14 @@ if(STRATA_NATIVE_EXPERTS)set(GGML_BACKEND_DL OFF CACHE BOOL "" FORCE)set(GGML_BUILD_TESTS OFF CACHE BOOL "" FORCE)set(GGML_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)- if(STRATA_PORTABLE)+ if(STRATA_PORTABLE AND NOT STRATA_CPU_X86)+ # arm64 (DGX Spark: Cortex-X925 + A725): ggml-cpu for any Armv8.2 CPU with dot product, FP16 and int8 matmul instead of the build+ # host's, so the engine built here does not depend on which core the compiler ran on. Without it GGML_NATIVE OFF would leave+ # ggml-cpu at plain armv8-a, and a native pack's CPU experts would lose the NEON dot products.+ set(GGML_NATIVE OFF CACHE BOOL "" FORCE)+ set(STRATA_ARM64_ARCH "armv8.2-a+dotprod+fp16+i8mm" CACHE STRING "arm64: ggml-cpu's -march for a STRATA_PORTABLE build")+ set(GGML_CPU_ARM_ARCH "${STRATA_ARM64_ARCH}" CACHE STRING "" FORCE)+ elseif(STRATA_PORTABLE)set(GGML_NATIVE OFF CACHE BOOL "" FORCE)foreach(_f GGML_AVX GGML_AVX2 GGML_FMA GGML_F16C GGML_BMI2)set(${_f} ON CACHE BOOL "" FORCE)@@ -1897,6 +1909,10 @@ if(STRATA_NATIVE_EXPERTS)src/kernels/cpu/kq_avx2.cpp src/kernels/cpu/kq_avx1.cpp src/kernels/cpu/q2_avx1.cppsrc/kernels/cpu/iq_avx1.cpp src/kernels/cpu/nvfp4_avx512.cpp src/kernels/cpu/nvfp4_avx2.cppsrc/kernels/cpu/tq_cpu.cpp src/kernels/cpu/tq_avx2.cpp src/kernels/cpu/tq_avx512.cpp)+ else()+ # the trellis dispatcher alone (the DeepSeek engine's CPU lane and Gyro): tq_cpu.cpp runs its scalar rows when the x86 units are+ # not compiled (the Metal build does the same)+ target_sources(strata_kernels_cpu PRIVATE src/kernels/cpu/tq_cpu.cpp)endif()target_include_directories(strata_kernels_cpu PRIVATE third_party/ggml)if(MSVC)@@ -1949,6 +1965,8 @@ if(STRATA_NATIVE_EXPERTS)target_link_libraries(dequant_q5_1_test PRIVATE strata_artifact ggml-base)add_test(NAME dequant_q5_1_test COMMAND dequant_q5_1_test)endif()+ # the x86 kernels' parity programs and benches (to the matching endif below): not on arm64, which has none of those kernels+ if(STRATA_CPU_X86)# the router lookahead's dot on each rung this CPU has (AVX2, the older CPUs' AVX1) against a double referenceadd_executable(router_dot_parity src/kernels/cpu/router_dot_parity.cpp)target_link_libraries(router_dot_parity PRIVATE strata_kernels_cpu)@@ -2009,6 +2027,7 @@ if(STRATA_NATIVE_EXPERTS)add_test(NAME q8k_quant_parity COMMAND q8k_quant_parity)add_test(NAME q8k_quant_parity_avx COMMAND ${CMAKE_COMMAND} -E env STRATA_FORCE_ISA=avx $<TARGET_FILE:q8k_quant_parity>)endif()+ endif() # STRATA_CPU_X86set(_strata_mmq_types q2_0 iq2_xxs iq2_xs iq2_s iq3_xxs iq3_s iq4_nl iq4_xs q8_0)# APR: the agention llama.cpp fork has MMQ instances for its trellis expert types (f62fdead0); any other ggml# tree leaves them to the FP16 prompt pathdiff --git a/spark/build_engine.sh b/spark/build_engine.shnew file mode 100755index 000000000..0b9a7f736--- /dev/null+++ b/spark/build_engine.sh@@ -0,0 +1,94 @@+#!/bin/sh+# Build the Strata CUDA engine for an NVIDIA DGX Spark (GB10, sm_121, arm64 Linux) into a ready-made engine folder.+#+# Runs on arm64 Linux with a CUDA 13 toolkit (13.0 is the release toolkit), on the Spark itself.+#+# spark/build_engine.sh # engine + DeepSeek engine + picture reader -> ./out-spark/engine+# BUILD_VISION=0 spark/build_engine.sh # text only (faster)+# BUILD_DSV41=0 spark/build_engine.sh # without the DeepSeek-V4.1 engine (strata-dsv41)+# ARCHS="121" OUT=/tmp/eng JOBS=8 spark/build_engine.sh+#+# The result is the folder setup.py keeps a ready-made engine in: copy it to Strata/engine on the Spark and run+# ./setup.sh there (docs/DGX_SPARK_LINUX.md). packaging/build_cuda_linux_aarch64.sh runs this and then makes the engine+# package for the signed feed. The flags are packaging/build_cuda_linux.sh's (the release build), for arm64:+# STRATA_PORTABLE makes ggml-cpu a baseline Armv8.2 build (dotprod + fp16 + i8mm, what a Grace CPU has) instead of the build+# host's, and both programs find the CUDA libraries beside themselves ($ORIGIN), never the toolkit they were built with.+set -eu+repo=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd -P)+archs=${ARCHS:-121}+out=${OUT:-$repo/out-spark/engine}+build=${BUILD:-$repo/build-spark}+vision_dir=${BUILD_VISION_DIR:-$repo/build-vision-spark}+jobs=${JOBS:-$(nproc)}+py=${PYTHON:-python3}+cmake=${CMAKE:-cmake}+llama=${STRATA_GGML_DIR:-$repo/third_party/llama.cpp}++case "$(uname -s)-$(uname -m)" in+ Linux-aarch64|Linux-arm64) ;;+ *) echo "build_engine.sh: this builds on arm64 Linux (the DGX Spark)" >&2; exit 2 ;;+esac+# DGX OS keeps the toolkit outside PATH for a plain user; the release build's toolkit is 13.0+if ! command -v nvcc >/dev/null 2>&1 && [ -x /usr/local/cuda/bin/nvcc ]; then+ PATH=/usr/local/cuda/bin:$PATH+ export PATH+fi+cudafull=$(nvcc --version | sed -n 's/.*release \([0-9][0-9]*\.[0-9][0-9]*\).*/\1/p' | head -1)+case "$cudafull" in+ 13.0|13.1) ;;+ 13.2) echo "build_engine.sh: warning: CUDA 13.2.0 / 13.2.1 miscompiled the engine for sm_120 (13.2.2 is fixed;" \+ "docs/TROUBLESHOOTING.md); 13.0 is the release toolkit" >&2 ;;+ 13.*) echo "build_engine.sh: note: CUDA $cudafull has not been tried for this build; 13.0 is the release toolkit" >&2 ;;+ *) echo "build_engine.sh: needs the CUDA 13 toolkit (nvcc here: ${cudafull:-missing})" >&2; exit 2 ;;+esac+if [ ! -f "$llama/ggml/CMakeLists.txt" ]; then+ # llama.cpp at the pinned commit (ggml for the engine, mtmd for pictures), exactly as setup.py fetches it+ (cd "$repo" && "$py" -c "import setup; print(setup.get_llama_cpp())")+ [ -f "$llama/ggml/CMakeLists.txt" ] || { echo "build_engine.sh: no llama.cpp tree at $llama after fetching it (STRATA_GGML_DIR=<tree> names another)" >&2; exit 2; }+fi+# the commit the DeepSeek engine prints with --version: a checkout gives it to CMake itself; a tree copy without .git needs COMMIT=<hex>+want_dsv41=OFF; [ "${BUILD_DSV41:-1}" = 1 ] && want_dsv41=ON+if [ "$want_dsv41" = ON ] && [ ! -e "$repo/.git" ] && [ ! -f "$repo/BUILD_ID" ]; then+ [ -n "${COMMIT:-}" ] || { echo "build_engine.sh: $repo has no .git and no BUILD_ID: set COMMIT=<hex> so the DeepSeek engine can print its commit" >&2; exit 2; }+ printf '%s\n' "$(printf '%s' "$COMMIT" | cut -c1-9)" > "$repo/BUILD_ID"+fi++real=""; for a in $(echo "$archs" | tr ';,' ' '); do real="$real;${a}-real"; done; real=${real#;}+last=$(echo "$archs" | tr ';,' ' ' | awk '{print $NF}')+cd "$repo"+"$cmake" -G Ninja -S . -B "$build" -DCMAKE_BUILD_TYPE=Release -DSTRATA_ENABLE_CUDA=ON -DSTRATA_BUILD_TESTS=OFF \+ -DSTRATA_NATIVE_EXPERTS=ON -DSTRATA_PORTABLE=ON -DSTRATA_Q6K_EXPERTS=ON -DSTRATA_MMQ_KQUANTS=ON \+ -DSTRATA_BUILD_DSV41="$want_dsv41" -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON -DCMAKE_SKIP_BUILD_RPATH=OFF '-DCMAKE_INSTALL_RPATH=$ORIGIN' \+ "-DCMAKE_CUDA_ARCHITECTURES=${real};${last}-virtual" -DSTRATA_GGML_DIR="$llama"+rm -f "$build/strata-dsv41"+if [ "$want_dsv41" = ON ]; then+ "$cmake" --build "$build" --target strata strata_kq strata-dsv41 -j "$jobs"+else+ "$cmake" --build "$build" --target strata strata_kq -j "$jobs"+fi++if [ "${BUILD_VISION:-1}" = 1 ]; then+ "$cmake" -G Ninja -S tools/vision -B "$vision_dir" -DCMAKE_BUILD_TYPE=Release -DLLAMA_DIR="$llama" \+ -DSTRATA_VISION_CUDA=ON -DSTRATA_PORTABLE=ON -DGGML_CUDA_NCCL=OFF -DGGML_CPU_ARM_ARCH=armv8.2-a+dotprod+fp16+i8mm \+ "-DCMAKE_CUDA_ARCHITECTURES=${real};${last}-virtual"+ "$cmake" --build "$vision_dir" --target strata-vision -j "$jobs"+fi++ver=$(sed -n 's/^[[:space:]]*project([^)]*VERSION[[:space:]]*\([0-9][0-9.]*\).*/\1/p' CMakeLists.txt | head -1)+rm -rf "$out"+mkdir -p "$out"+cp "$build/strata" "$out/"+[ -f "$build/libstrata_kq.so" ] && cp "$build/libstrata_kq.so" "$out/"+# DeepSeek-V4.1's engine beside it: setup.py takes engine/strata-dsv41 (dsv41_exe)+[ "$want_dsv41" = ON ] && [ -f "$build/strata-dsv41" ] && cp "$build/strata-dsv41" "$out/"+vision_json=""+if [ "${BUILD_VISION:-1}" = 1 ] && [ -f "$vision_dir/bin/strata-vision" ]; then+ cp "$vision_dir/bin/strata-vision" "$out/"+ vision_json=", \"vision_archs\": \"$archs\""+fi+# the CUDA libraries the engine loads (cuBLAS, cuBLASLt, cudart) go beside it: the Spark needs only its driver+"$py" packaging/collect_libs.py --binary "$build/strata" --search /usr/local/cuda/lib64 --out "$out" --also 'libcublasLt.so*'+echo "{\"version\": \"$ver\", \"source\": \"spark/build_engine.sh\", \"platform\": \"linux-aarch64\", \"archs\": \"$archs\", \"ptx\": true, \"cuda\": \"$cudafull\", \"portable\": true$vision_json}" > "$out/BUILD.json"+"$out/strata" --version 2>/dev/null || true+[ "$want_dsv41" = ON ] && { "$out/strata-dsv41" --version 2>/dev/null || true; }+echo "built: $out"diff --git a/src/dsv41/ds4_tq_cpu.cpp b/src/dsv41/ds4_tq_cpu.cppindex afafb584a..7aefb6c82 100644--- a/src/dsv41/ds4_tq_cpu.cpp+++ b/src/dsv41/ds4_tq_cpu.cpp@@ -53,8 +53,10 @@ void dots(int type, const uint8_t* w, size_t rb, int n, const void* const* act,const int kb = kc::tq_kbits(type);const int16_t* tab = kc::tq_state_table_for(type);switch (kc::tq_isa()) {+#if defined(__x86_64__) || defined(_M_X64) // the AVX units exist on x86-64 builds only (arm64: the scalar rows)case kc::TqIsa::avx512: kc::tq_detail::tq_rows_avx512(kb, tab, w, rb, n, ap, nt, out, r0, r1); break;case kc::TqIsa::avx2: kc::tq_detail::tq_rows_avx2(kb, tab, w, rb, n, ap, nt, out, r0, r1); break;+#endifdefault: kc::tq_detail::tq_rows_scalar(kb, tab, w, rb, n, ap, nt, out, r0, r1); break;}}diff --git a/src/kernels/cpu/portable.cpp b/src/kernels/cpu/portable.cppindex a8ff501cc..09cc3271a 100644--- a/src/kernels/cpu/portable.cpp+++ b/src/kernels/cpu/portable.cpp@@ -15,7 +15,6 @@#include "strata/kernels/cpu/iq_avx1.hpp"#include "strata/kernels/cpu/nvfp4_avx2.hpp"#include "strata/kernels/cpu/nvfp4_avx512.hpp"-#include "strata/kernels/cpu/tq_cpu.hpp"#include <atomic>#include <cmath>@@ -203,12 +202,6 @@ void nvfp4_512_rows(const uint8_t* w, size_t row_bytes, int n, const void* constvoid q2_0_gguf_rows_multi_avx1_legacy(const uint8_t* w, size_t row_bytes, int nblocks, const ActQ* const* a, int nt, float* const* out, int r0, int r1) { cpu_kernel_unreachable("q2_0_gguf_rows_multi_avx1_legacy"); }void q2_0_gguf_rows_multi_avx1(const uint8_t* w, size_t row_bytes, int nblocks, const ActQ* const* a, int nt, float* const* out, int r0, int r1) { cpu_kernel_unreachable("q2_0_gguf_rows_multi_avx1"); }void q8k_quant_avx1(const float* x, void* y, int64_t n) { cpu_kernel_unreachable("q8k_quant_avx1"); }-void tq_gu_rows(TqIsa isa, int ggml_type, const uint8_t* blob, size_t gu_row, size_t up_off, int n, const void* const* act, int nt, float* const* ff, int r0, int r1) { cpu_kernel_unreachable("tq_gu_rows"); }-TqIsa tq_isa() { return TqIsa::scalar; }-int tq_kbits(int ggml_type) noexcept { return 0; }-bool tq_kernels_enabled() { return false; }-void tq_rows(TqIsa isa, int ggml_type, const uint8_t* w, size_t row_bytes, int n, const void* const* act, int nt, float* const* out, int r0, int r1) { cpu_kernel_unreachable("tq_rows"); }--bool tq_supported(int) noexcept { return false; }+// (the trellis rows are tq_cpu.cpp's: its scalar kernel runs here, as in the Metal build)} // namespace strata::kernels::cpu