Skip to content

Distribute and run LLMs with a single file.

Add tinyblasStrsmBatched kernel (#923)

Davide Eynardeb7f66cefb6d
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_solve_tri.cu.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_solve_tri.cu.patch
deleted file mode 100644
index 81949ffa..00000000
--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_solve_tri.cu.patch
+++ /dev/null
@@ -1,42 +0,0 @@
-diff --git a/ggml/src/ggml-cuda/solve_tri.cu b/ggml/src/ggml-cuda/solve_tri.cu
---- a/llama.cpp/ggml/src/ggml-cuda/solve_tri.cu
-+++ b/llama.cpp/ggml/src/ggml-cuda/solve_tri.cu
-@@ -5,6 +5,10 @@
- #define MAX_N_FAST 64
- #define MAX_K_FAST 32
-
-+// cuBLAS TRSM-based implementation for large matrices
-+// Not available when using TinyBLAS (which doesn't implement TRSM)
-+#ifndef GGML_TINYBLAS_NO_TRSM
-+
- static __global__ void get_batch_pointers(const float * A,
- float * X,
- const float ** A_ptrs,
-@@ -78,6 +82,8 @@ static void solve_tri_f32_cublas(ggml_backend_cuda_context & ctx,
- GGML_UNUSED_VARS(s12, s13);
- }
-
-+#endif // GGML_TINYBLAS_NO_TRSM
-+
- // ======================
- // Fast Kernel (n <= 64, k <= 32) - Warp-based parallel reduction
- // ======================
-@@ -267,9 +273,18 @@ void ggml_cuda_op_solve_tri(ggml_backend_cuda_context & ctx, ggml_tensor * dst)
- src1->nb[2] / sizeof(float), src1->nb[3] / sizeof(float), dst->nb[2] / sizeof(float),
- dst->nb[3] / sizeof(float), ctx.stream());
- } else {
-+#ifdef GGML_TINYBLAS_NO_TRSM
-+ // TinyBLAS doesn't support TRSM (triangular solve)
-+ // This path is only reached by Qwen3-Next models with large matrices
-+ GGML_ABORT("solve_tri with n > %d or k > %d requires cuBLAS TRSM which is not available with TinyBLAS. "
-+ "This operation is only used by Qwen3-Next models. "
-+ "Please rebuild with cuBLAS or use CPU backend for this model.",
-+ MAX_N_FAST, MAX_K_FAST);
-+#else
- solve_tri_f32_cublas(ctx, (const float *) src0->data, (const float *) src1->data, (float *) dst->data, n, k,
- ne02, ne03, src0->nb[2] / sizeof(float), src0->nb[3] / sizeof(float),
- src1->nb[2] / sizeof(float), src1->nb[3] / sizeof(float), dst->nb[2] / sizeof(float),
- dst->nb[3] / sizeof(float), ctx.stream());
-+#endif // GGML_TINYBLAS_NO_TRSM
- }
- }
diff --git a/llamafile/tinyblas-compat.h b/llamafile/tinyblas-compat.h
index ca01813d..7a9ee7d8 100644
--- a/llamafile/tinyblas-compat.h
+++ b/llamafile/tinyblas-compat.h
@@ -128,25 +128,28 @@
#define CUBLAS_DEFAULT_MATH 0
// ============================================================================
-// Triangular solve (TRSM) - Not supported in TinyBLAS
-//
-// cublasStrsmBatched is used only by solve_tri.cu for Qwen3-Next model's
-// gated delta net attention. When TinyBLAS is used, the CUDA backend will
-// fall back to CPU for this operation.
+// Triangular solve (TRSM) type mappings
// ============================================================================
-#define GGML_TINYBLAS_NO_TRSM 1
+#define cublasSideMode_t tinyblasSideMode_t
+#define cublasFillMode_t tinyblasFillMode_t
+#define cublasDiagType_t tinyblasDiagType_t
+
+// ============================================================================
+// TRSM constant mappings
+// ============================================================================
-// Define TRSM-related constants so code compiles, but operations will fail
-#define CUBLAS_SIDE_LEFT 0
-#define CUBLAS_SIDE_RIGHT 1
-#define CUBLAS_FILL_MODE_LOWER 0
-#define CUBLAS_FILL_MODE_UPPER 1
-#define CUBLAS_DIAG_NON_UNIT 0
-#define CUBLAS_DIAG_UNIT 1
+#define CUBLAS_SIDE_LEFT TINYBLAS_SIDE_LEFT
+#define CUBLAS_SIDE_RIGHT TINYBLAS_SIDE_RIGHT
+#define CUBLAS_FILL_MODE_LOWER TINYBLAS_FILL_MODE_LOWER
+#define CUBLAS_FILL_MODE_UPPER TINYBLAS_FILL_MODE_UPPER
+#define CUBLAS_DIAG_NON_UNIT TINYBLAS_DIAG_NON_UNIT
+#define CUBLAS_DIAG_UNIT TINYBLAS_DIAG_UNIT
+
+// ============================================================================
+// TRSM function mapping
+// ============================================================================
-// Stub for cublasStrsmBatched - always returns not supported
-// The solve_tri.cu code should check GGML_TINYBLAS_NO_TRSM and fall back to CPU
-#define cublasStrsmBatched(...) TINYBLAS_STATUS_NOT_SUPPORTED
+#define cublasStrsmBatched tinyblasStrsmBatched
#endif // GGML_USE_TINYBLAS
diff --git a/llamafile/tinyblas.cu b/llamafile/tinyblas.cu
index 1d4da084..f1550f49 100644
--- a/llamafile/tinyblas.cu
+++ b/llamafile/tinyblas.cu
@@ -979,3 +979,65 @@ tinyblasStatus_t tinyblasGemmStridedBatchedEx(tinyblasHandle_t handle, //
return TINYBLAS_STATUS_INVALID_VALUE;
}
}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+// TRSM: Triangular Solve (Batched)
+//
+// Solves X * A = alpha * B where A is upper triangular, writing result into B.
+// One thread block per batch element. Threads parallelize over m rows.
+
+static __global__ void tinyblasStrsmBatched_kernel(int m, int n, float alpha,
+ const float *const *Aarray, int lda,
+ float *const *Barray, int ldb) {
+ const int batch = blockIdx.x;
+ const float *A = Aarray[batch];
+ float *B = Barray[batch];
+
+ // Solve X * A = alpha * B for right-side upper triangular (non-unit diagonal).
+ // Process columns left-to-right. For column j:
+ // B[i][j] = (alpha * B[i][j] - sum_{p<j} B[i][p] * A[p][j]) / A[j][j]
+ for (int j = 0; j < n; j++) {
+ // Column-major: A[p][j] = A[p + j*lda], B[i][j] = B[i + j*ldb]
+ float diag = A[j + j * lda];
+ __syncthreads();
+ for (int i = threadIdx.x; i < m; i += blockDim.x) {
+ float val = alpha * B[i + j * ldb];
+ for (int p = 0; p < j; p++) {
+ val -= B[i + p * ldb] * A[p + j * lda];
+ }
+ B[i + j * ldb] = val / diag;
+ }
+ }
+}
+
+static tinyblasStatus_t tinyblasStrsmBatched_launch(tinyblasHandle_t handle, int m, int n,
+ float alpha, const float *const *Aarray,
+ int lda, float *const *Barray, int ldb,
+ int batchCount) {
+ int threads = std::min(1024, ((m + 31) / 32) * 32);
+ if (threads < 1)
+ threads = 1;
+ tinyblasStrsmBatched_kernel<<<batchCount, threads, 0, handle->stream>>>(m, n, alpha, Aarray,
+ lda, Barray, ldb);
+ cudaError_t err = cudaGetLastError();
+ if (err != cudaSuccess)
+ return TINYBLAS_STATUS_EXECUTION_FAILED;
+ return TINYBLAS_STATUS_SUCCESS;
+}
+
+tinyblasStatus_t tinyblasStrsmBatched(tinyblasHandle_t handle, tinyblasSideMode_t side,
+ tinyblasFillMode_t uplo, tinyblasOperation_t trans,
+ tinyblasDiagType_t diag, int m, int n, const float *alpha,
+ const float *const Aarray[], int lda, float *const Barray[],
+ int ldb, int batchCount) {
+ if (side != TINYBLAS_SIDE_RIGHT || uplo != TINYBLAS_FILL_MODE_UPPER ||
+ trans != TINYBLAS_OP_N || diag != TINYBLAS_DIAG_NON_UNIT)
+ return TINYBLAS_STATUS_NOT_SUPPORTED;
+ if (!handle || !alpha || !Aarray || !Barray)
+ return TINYBLAS_STATUS_INVALID_VALUE;
+ if (m < 0 || n < 0 || lda < n || ldb < m)
+ return TINYBLAS_STATUS_INVALID_VALUE;
+ if (batchCount == 0)
+ return TINYBLAS_STATUS_SUCCESS;
+ return tinyblasStrsmBatched_launch(handle, m, n, *alpha, Aarray, lda, Barray, ldb, batchCount);
+}
diff --git a/llamafile/tinyblas.h b/llamafile/tinyblas.h
index d6c2aae1..6fb6214a 100644
--- a/llamafile/tinyblas.h
+++ b/llamafile/tinyblas.h
@@ -7,6 +7,21 @@ typedef enum tinyblasOperation {
TINYBLAS_OP_T,
} tinyblasOperation_t;
+typedef enum tinyblasSideMode {
+ TINYBLAS_SIDE_LEFT,
+ TINYBLAS_SIDE_RIGHT,
+} tinyblasSideMode_t;
+
+typedef enum tinyblasFillMode {
+ TINYBLAS_FILL_MODE_LOWER,
+ TINYBLAS_FILL_MODE_UPPER,
+} tinyblasFillMode_t;
+
+typedef enum tinyblasDiagType {
+ TINYBLAS_DIAG_NON_UNIT,
+ TINYBLAS_DIAG_UNIT,
+} tinyblasDiagType_t;
+
typedef enum tinyblasDataType {
TINYBLAS_R_32F,
TINYBLAS_R_16F,
@@ -45,6 +60,11 @@ tinyblasStatus_t tinyblasSgemm(tinyblasHandle_t, tinyblasOperation_t, tinyblasOp
int, const float *, const float *, int, const float *, int,
const float *, float *, int);
+tinyblasStatus_t tinyblasStrsmBatched(tinyblasHandle_t, tinyblasSideMode_t, tinyblasFillMode_t,
+ tinyblasOperation_t, tinyblasDiagType_t, int, int,
+ const float *, const float *const[], int,
+ float *const[], int, int);
+
tinyblasStatus_t tinyblasGemmEx(tinyblasHandle_t, tinyblasOperation_t, tinyblasOperation_t, int,
int, int, const void *, const void *, tinyblasDataType_t, int,
const void *, tinyblasDataType_t, int, const void *, void *,
diff --git a/tests/BUILD.mk b/tests/BUILD.mk
index 7d6788eb..fb1fe7ba 100644
--- a/tests/BUILD.mk
+++ b/tests/BUILD.mk
@@ -4,6 +4,7 @@
PKGS += TESTS
include tests/sgemm/BUILD.mk
+include tests/strsm/BUILD.mk
# ==============================================================================
# Include paths (reuse llamafile includes)
diff --git a/tests/strsm/BUILD.mk b/tests/strsm/BUILD.mk
new file mode 100644
index 00000000..8172ea50
--- /dev/null
+++ b/tests/strsm/BUILD.mk
@@ -0,0 +1,22 @@
+#-*-mode:makefile-gmake;indent-tabs-mode:t;tab-width:8;coding:utf-8-*-┐
+#── vi: set noet ft=make ts=8 sw=8 fenc=utf-8 :vi ────────────────────┘
+
+# ==============================================================================
+# STRSM Manual Tests (TinyBLAS vs cuBLAS)
+# ==============================================================================
+# These are manual CUDA tests for validating tinyblasStrsmBatched correctness
+# and performance against cuBLAS. They are NOT included in `make check` because:
+# - They require an NVIDIA GPU with CUDA toolkit installed
+# - They require cuBLAS (libcublas.so) for the reference implementation
+# - They are compiled with nvcc, not cosmocc
+# - They are primarily for development validation
+#
+# To build and run:
+# bash tests/strsm/build_and_run.sh
+#
+# To build only:
+# bash tests/strsm/build_and_run.sh --build-only
+#
+# The test compiles both tinyblas.cu and cublas into one binary, running the
+# same STRSM operation through both backends and comparing results directly
+# on the GPU.
diff --git a/tests/strsm/build_and_run.sh b/tests/strsm/build_and_run.sh
new file mode 100755
index 00000000..6da3e8e3
--- /dev/null
+++ b/tests/strsm/build_and_run.sh
@@ -0,0 +1,148 @@
+#!/bin/bash
+# -*- mode:sh;indent-tabs-mode:nil;tab-width:4;coding:utf-8 -*-
+# vi: set et ft=sh ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#
+# Copyright 2026 Mozilla.ai
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Build and run STRSM batched test: TinyBLAS vs cuBLAS
+#
+# Usage:
+# bash tests/strsm/build_and_run.sh # Build and run
+# bash tests/strsm/build_and_run.sh --build-only # Build only
+# bash tests/strsm/build_and_run.sh --clean # Clean and rebuild
+#
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+REPO_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
+LLAMAFILE_DIR="$REPO_DIR/llamafile"
+OUTPUT_DIR="$REPO_DIR/o/tests/strsm"
+OUTPUT="$OUTPUT_DIR/strsm_test"
+
+CUDA_PATH="${CUDA_PATH:-/usr/local/cuda}"
+NVCC="${CUDA_PATH}/bin/nvcc"
+
+BUILD_ONLY=0
+CLEAN=0
+
+for arg in "$@"; do
+ case "$arg" in
+ --build-only)
+ BUILD_ONLY=1
+ ;;
+ --clean)
+ CLEAN=1
+ ;;
+ --help)
+ echo "Usage: $0 [--build-only] [--clean]"
+ echo " --build-only Build without running"
+ echo " --clean Clean and rebuild"
+ exit 0
+ ;;
+ *)
+ echo "Unknown option: $arg"
+ exit 1
+ ;;
+ esac
+done
+
+# Check nvcc
+if [ ! -x "$NVCC" ]; then
+ echo "Error: nvcc not found at $NVCC"
+ echo "Install CUDA toolkit or set CUDA_PATH"
+ exit 1
+fi
+
+# Check cublas
+CUBLAS_LIB=""
+for dir in "$CUDA_PATH/lib64" "$CUDA_PATH/lib"; do
+ if [ -f "$dir/libcublas.so" ]; then
+ CUBLAS_LIB="$dir"
+ break
+ fi
+done
+if [ -z "$CUBLAS_LIB" ]; then
+ echo "Error: libcublas.so not found in $CUDA_PATH/lib64 or $CUDA_PATH/lib"
+ exit 1
+fi
+
+# Auto-detect GPU architecture
+detect_gpu_arch() {
+ if command -v nvidia-smi &>/dev/null; then
+ # Get compute capability from nvidia-smi
+ local cc=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d '.')
+ if [ -n "$cc" ] && [ "$cc" -gt 0 ] 2>/dev/null; then
+ echo "$cc"
+ return
+ fi
+ fi
+ # Default: compile for common architectures
+ echo ""
+}
+
+GPU_ARCH=$(detect_gpu_arch)
+if [ -n "$GPU_ARCH" ]; then
+ ARCH_FLAGS="-gencode arch=compute_${GPU_ARCH},code=sm_${GPU_ARCH}"
+ echo "Detected GPU: sm_${GPU_ARCH}"
+else
+ ARCH_FLAGS="-gencode arch=compute_75,code=sm_75 \
+ -gencode arch=compute_80,code=sm_80 \
+ -gencode arch=compute_86,code=sm_86 \
+ -gencode arch=compute_89,code=sm_89 \
+ -gencode arch=compute_90,code=sm_90"
+ echo "No GPU detected, compiling for sm_75/80/86/89/90"
+fi
+
+# Clean
+if [ "$CLEAN" = "1" ] && [ -d "$OUTPUT_DIR" ]; then
+ echo "Cleaning..."
+ rm -rf "$OUTPUT_DIR"
+fi
+
+mkdir -p "$OUTPUT_DIR"
+
+# Copy tinyblas files to build dir to avoid include path conflicts.
+# The llamafile/ directory has its own string.h that clashes with system headers
+# when added via -I. This is the same approach used by cuda.sh.
+cp "$LLAMAFILE_DIR/tinyblas.h" "$OUTPUT_DIR/"
+cp "$LLAMAFILE_DIR/tinyblas.cu" "$OUTPUT_DIR/"
+
+# Build
+echo "Building strsm_test..."
+echo " Sources: strsm_test.cu + tinyblas.cu"
+echo " Output: $OUTPUT"
+
+$NVCC \
+ "$SCRIPT_DIR/strsm_test.cu" \
+ "$OUTPUT_DIR/tinyblas.cu" \
+ -I "$OUTPUT_DIR" \
+ -o "$OUTPUT" \
+ $ARCH_FLAGS \
+ -lcublas -lcuda \
+ --use_fast_math \
+ --extended-lambda \
+ -O2 \
+ -std=c++17
+
+echo "Build successful: $OUTPUT"
+echo ""
+
+# Run
+if [ "$BUILD_ONLY" = "0" ]; then
+ echo "Running..."
+ echo ""
+ "$OUTPUT"
+fi
diff --git a/tests/strsm/strsm_test.cu b/tests/strsm/strsm_test.cu
new file mode 100644
index 00000000..d65c3862
--- /dev/null
+++ b/tests/strsm/strsm_test.cu
@@ -0,0 +1,740 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2026 Mozilla.ai
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//
+// ============================================================================
+// strsm_test: TinyBLAS vs cuBLAS StrsmBatched Comparison
+// ============================================================================
+//
+// PURPOSE:
+// Validates tinyblasStrsmBatched correctness by comparing against cuBLAS
+// (our ground truth). Both implementations run in the same process on the
+// same GPU data for a direct comparison.
+//
+// OPERATION:
+// Solves X * A = alpha * B where A is upper triangular (SIDE_RIGHT,
+// FILL_MODE_UPPER, OP_N, DIAG_NON_UNIT). Result overwrites B in-place.
+// This is the configuration used by ggml's solve_tri CUDA backend.
+//
+// BUILD:
+// bash tests/strsm/build_and_run.sh
+//
+// METRICS:
+// - Time (us): Average kernel execution time over multiple iterations
+// - Max abs error: Largest element-wise absolute difference
+// - RMSE: Root mean square error across all elements
+// - Max ULP diff: Largest Units in Last Place difference
+//
+
+#include <cublas_v2.h>
+#include <cuda_runtime.h>
+#include "tinyblas.h"
+
+#include <cfloat>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <ctime>
+#include <vector>
+
+// ============================================================================
+// Error checking macros
+// ============================================================================
+
+#define CUDA_CHECK(call) \
+ do { \
+ cudaError_t err = (call); \
+ if (err != cudaSuccess) { \
+ fprintf(stderr, "CUDA error at %s:%d: %s\n", __FILE__, __LINE__, \
+ cudaGetErrorString(err)); \
+ exit(1); \
+ } \
+ } while (0)
+
+#define CUBLAS_CHECK(call) \
+ do { \
+ cublasStatus_t err = (call); \
+ if (err != CUBLAS_STATUS_SUCCESS) { \
+ fprintf(stderr, "cuBLAS error at %s:%d: %d\n", __FILE__, __LINE__, \
+ (int)err); \
+ exit(1); \
+ } \
+ } while (0)
+
+#define TINYBLAS_CHECK(call) \
+ do { \
+ tinyblasStatus_t err = (call); \
+ if (err != TINYBLAS_STATUS_SUCCESS) { \
+ fprintf(stderr, "TinyBLAS error at %s:%d: %s\n", __FILE__, \
+ __LINE__, tinyblasGetStatusString(err)); \
+ exit(1); \
+ } \
+ } while (0)
+
+// ============================================================================
+// Utilities
+// ============================================================================
+
+static inline long long micros(void) {
+ struct timespec ts;
+ clock_gettime(CLOCK_REALTIME, &ts);
+ return ts.tv_sec * 1000000LL + (ts.tv_nsec + 999) / 1000;
+}
+
+static unsigned rand_state = 42;
+
+static inline unsigned rand32(void) {
+ rand_state = rand_state * 1664525u + 1013904223u;
+ return rand_state;
+}
+
+static inline float randf(void) {
+ // Random float in [-1, 1]
+ return (float)(rand32() & 0xFFFF) / 32768.0f - 1.0f;
+}
+
+static inline unsigned float_to_bits(float f) {
+ unsigned bits;
+ memcpy(&bits, &f, sizeof(bits));
+ return bits;
+}
+
+static inline long long ulp_diff(float a, float b) {
+ // ULP distance between two floats
+ unsigned ai = float_to_bits(a);
+ unsigned bi = float_to_bits(b);
+ // Handle sign: convert to two's complement-like representation
+ if (ai & 0x80000000u)
+ ai = 0x80000000u - ai;
+ if (bi & 0x80000000u)
+ bi = 0x80000000u - bi;
+ long long diff = (long long)ai - (long long)bi;
+ return diff < 0 ? -diff : diff;
+}
+
+// ============================================================================
+// Residual computation (host-side, double precision)
+// ============================================================================
+
+// Compute the scaled backward error for X*A = alpha*B:
+// ||X*A - alpha*B||_inf / (||A||_inf * ||X||_inf * n * eps)
+//
+// A is n x n upper triangular (column-major, lda stride)
+// X and B are m x n (column-major, ldb stride)
+// All inputs are float but computation is in double for accuracy.
+// Returns the scaled residual (< 1.0 means good, > 10 means trouble).
+static double compute_residual(const float *X, const float *A, const float *B_orig,
+ int m, int n, int lda, int ldb, float alpha) {
+ const double eps = 1.1920929e-7; // float32 machine epsilon
+
+ // Compute ||A||_inf = max row sum of |A|
+ double norm_A = 0.0;
+ for (int i = 0; i < n; i++) {
+ double row_sum = 0.0;
+ for (int j = 0; j < n; j++)
+ row_sum += fabs((double)A[i + j * lda]);
+ if (row_sum > norm_A) norm_A = row_sum;
+ }
+
+ // Compute ||X||_inf = max row sum of |X|
+ double norm_X = 0.0;
+ for (int i = 0; i < m; i++) {
+ double row_sum = 0.0;
+ for (int j = 0; j < n; j++)
+ row_sum += fabs((double)X[i + j * ldb]);
+ if (row_sum > norm_X) norm_X = row_sum;
+ }
+
+ // Compute R = X*A - alpha*B, then ||R||_inf
+ // R[i][j] = sum_p(X[i][p] * A[p][j]) - alpha*B[i][j]
+ // A is upper triangular so A[p][j] = 0 for p > j
+ double norm_R = 0.0;
+ for (int i = 0; i < m; i++) {
+ double row_sum = 0.0;
+ for (int j = 0; j < n; j++) {
+ double xa_ij = 0.0;
+ // A is upper triangular: only p <= j contributes
+ for (int p = 0; p <= j; p++)
+ xa_ij += (double)X[i + p * ldb] * (double)A[p + j * lda];
+ double r_ij = xa_ij - (double)alpha * (double)B_orig[i + j * ldb];
+ row_sum += fabs(r_ij);
+ }
+ if (row_sum > norm_R) norm_R = row_sum;
+ }
+
+ double denom = norm_A * norm_X * n * eps;
+ if (denom == 0.0) return 0.0;
+ return norm_R / denom;
+}
+
+// ============================================================================
+// Matrix generation (column-major)
+// ============================================================================
+
+// Generate a random upper triangular matrix (n x n, column-major, lda stride)
+// Diagonal elements are guaranteed to be well-conditioned (away from zero)
+static void gen_upper_triangular(float *A, int n, int lda, bool identity = false,
+ float diag_min = 0.5f, float diag_max = 2.0f) {
+ // Zero the whole thing first
+ for (int j = 0; j < n; j++)
+ for (int i = 0; i < lda; i++)
+ A[j * lda + i] = 0.0f;
+
+ for (int j = 0; j < n; j++) {
+ // Upper triangle: A[i][j] where i <= j, stored at A[i + j*lda]
+ for (int i = 0; i <= j; i++) {
+ if (i == j) {
+ if (identity) {
+ A[i + j * lda] = 1.0f;
+ } else {
+ // Diagonal: random in [diag_min, diag_max] with random sign
+ float val = diag_min + (diag_max - diag_min) * fabsf(randf());
+ A[i + j * lda] = (rand32() & 1) ? val : -val;
+ }
+ } else {
+ A[i + j * lda] = identity ? 0.0f : randf();
+ }
+ }
+ }
+}
+
+// Generate an ill-conditioned upper triangular matrix
+// Diagonal elements span a wide range (diag_min to diag_max)
+static void gen_ill_conditioned(float *A, int n, int lda,
+ float diag_min = 1e-3f, float diag_max = 1e3f) {
+ for (int j = 0; j < n; j++)
+ for (int i = 0; i < lda; i++)
+ A[j * lda + i] = 0.0f;
+
+ for (int j = 0; j < n; j++) {
+ for (int i = 0; i <= j; i++) {
+ if (i == j) {
+ // Logarithmic spacing of diagonal elements
+ float t = (n > 1) ? (float)j / (float)(n - 1) : 0.5f;
+ float val = diag_min * powf(diag_max / diag_min, t);
+ A[i + j * lda] = (rand32() & 1) ? val : -val;
+ } else {
+ A[i + j * lda] = randf() * 0.1f; // Small off-diagonal
+ }
+ }
+ }
+}
+
+// Generate a random dense matrix (m x n, column-major, ld stride)
+static void gen_random_matrix(float *B, int m, int n, int ld) {
+ for (int j = 0; j < n; j++) {
+ for (int i = 0; i < ld; i++) {
+ B[j * ld + i] = (i < m) ? randf() : 0.0f;
+ }
+ }
+}
+
+// Generate matrices mimicking Qwen3.5 delta-net attention solve_tri usage.
+//
+// In the model, solve_tri receives:
+// A = I + lower_tri(K^T * K_b * decay_mask) (GGML lower triangular)
+// B = -lower_tri(K^T * K_b * decay_mask) (GGML lower triangular)
+//
+// Due to GGML row-major -> cuBLAS column-major transposition:
+// A appears as upper triangular with diagonal = 1.0
+// B appears as upper triangular with diagonal = 0.0
+//
+// The decay mask applies exponential decay: entry[i,j] *= exp(-rate * |i-j|)
+// so off-diagonal values shrink away from the diagonal.
+//
+// A_out: n x n upper triangular, lda stride, column-major
+// B_out: m x n dense (but structured as upper tri + noise), ldb stride
+static void gen_qwen_attention_matrices(float *A_out, float *B_out,
+ int n, int m, int lda, int ldb,
+ float decay_rate = 0.1f) {
+ // Zero everything
+ for (int j = 0; j < n; j++) {
+ for (int i = 0; i < lda; i++)
+ A_out[j * lda + i] = 0.0f;
+ for (int i = 0; i < ldb; i++)
+ B_out[j * ldb + i] = 0.0f;
+ }
+
+ // Build upper triangular A with diagonal = 1.0 and decaying off-diagonal
+ for (int j = 0; j < n; j++) {
+ for (int i = 0; i <= j; i++) {
+ if (i == j) {
+ A_out[i + j * lda] = 1.0f;
+ // B diagonal is 0 (from -lower_tri which excludes diagonal)
+ } else {
+ // Off-diagonal: attention-like values with exponential decay
+ float attn_val = randf() * 0.5f; // attention scores in [-0.5, 0.5]
+ float decay = expf(-decay_rate * (float)(j - i));
+ float val = attn_val * decay;
+ A_out[i + j * lda] = val; // A = I + upper_tri_part
+ // B = -(upper_tri_part), so B has negated off-diagonal
+ if (i < m) {
+ B_out[i + j * ldb] = -val;
+ }
+ }
+ }
+ // B also has content in rows beyond the triangle (from attention patterns)
+ for (int i = j + 1; i < m; i++) {
+ B_out[i + j * ldb] = randf() * 0.3f;
+ }
+ }
+}
+
+// ============================================================================
+// GPU batch pointer setup kernel
+// ============================================================================
+
+__global__ void setup_batch_pointers(const float **A_ptrs, float **B_ptrs,
+ const float *A_base, float *B_base,
+ int A_stride, int B_stride, int batch) {
+ int idx = blockIdx.x * blockDim.x + threadIdx.x;
+ if (idx < batch) {
+ A_ptrs[idx] = A_base + idx * A_stride;
+ B_ptrs[idx] = B_base + idx * B_stride;
+ }
+}
+
+// ============================================================================
+// Test configuration
+// ============================================================================
+
+struct TestConfig {
+ const char *name;
+ int n; // Size of triangular matrix A (n x n)
+ int m; // Number of rows in B (B is m x n)
+ int batch; // Batch count
+ float alpha; // Scalar multiplier
+ bool identity; // Use identity matrix for A
+ bool ill_cond; // Use ill-conditioned matrix
+ bool qwen; // Use Qwen-like attention matrices (I + decay*attn)
+ int iterations; // Timing iterations
+};
+
+struct TestResult {
+ bool pass;
+ double max_abs_err; // Max |cublas - tinyblas| element-wise
+ double rmse; // RMSE of cublas vs tinyblas
+ long long max_ulp; // Max ULP diff cublas vs tinyblas
+ double cublas_residual; // ||X_cublas * A - alpha*B|| / (||A||*||X||*n*eps)
+ double tinyblas_residual; // ||X_tiny * A - alpha*B|| / (||A||*||X||*n*eps)
+ double cublas_us;
+ double tinyblas_us;
+};
+
+// ============================================================================
+// Run a single test case
+// ============================================================================
+
+static TestResult run_test(const TestConfig &cfg) {
+ TestResult result = {};
+ const int n = cfg.n;
+ const int m = cfg.m;
+ const int batch = cfg.batch;
+ const float alpha = cfg.alpha;
+ const int lda = n; // Leading dimension of A (column-major, n x n)
+ const int ldb = m; // Leading dimension of B (column-major, m x n)
+
+ // ---- Host allocation ----
+ size_t A_size = (size_t)lda * n; // One A matrix
+ size_t B_size = (size_t)ldb * n; // One B matrix
+ size_t A_total = A_size * batch;
+ size_t B_total = B_size * batch;
+
+ std::vector<float> h_A(A_total);
+ std::vector<float> h_B(B_total);
+
+ // Generate matrices for each batch element
+ for (int b = 0; b < batch; b++) {
+ if (cfg.qwen) {
+ gen_qwen_attention_matrices(h_A.data() + b * A_size,
+ h_B.data() + b * B_size,
+ n, m, lda, ldb);
+ } else {
+ if (cfg.identity) {
+ gen_upper_triangular(h_A.data() + b * A_size, n, lda, /*identity=*/true);
+ } else if (cfg.ill_cond) {
+ gen_ill_conditioned(h_A.data() + b * A_size, n, lda);
+ } else {
+ gen_upper_triangular(h_A.data() + b * A_size, n, lda);
+ }
+ gen_random_matrix(h_B.data() + b * B_size, m, n, ldb);
+ }
+ }
+
+ // ---- Device allocation ----
+ float *d_A, *d_B_cublas, *d_B_tinyblas;
+ CUDA_CHECK(cudaMalloc(&d_A, A_total * sizeof(float)));
+ CUDA_CHECK(cudaMalloc(&d_B_cublas, B_total * sizeof(float)));
+ CUDA_CHECK(cudaMalloc(&d_B_tinyblas, B_total * sizeof(float)));
+
+ CUDA_CHECK(cudaMemcpy(d_A, h_A.data(), A_total * sizeof(float), cudaMemcpyHostToDevice));
+ CUDA_CHECK(cudaMemcpy(d_B_cublas, h_B.data(), B_total * sizeof(float), cudaMemcpyHostToDevice));
+ CUDA_CHECK(cudaMemcpy(d_B_tinyblas, h_B.data(), B_total * sizeof(float), cudaMemcpyHostToDevice));
+
+ // ---- Batch pointer arrays on device ----
+ const float **d_A_ptrs_cublas, **d_A_ptrs_tiny;
+ float **d_B_ptrs_cublas, **d_B_ptrs_tiny;
+ CUDA_CHECK(cudaMalloc(&d_A_ptrs_cublas, batch * sizeof(float *)));
+ CUDA_CHECK(cudaMalloc(&d_B_ptrs_cublas, batch * sizeof(float *)));
+ CUDA_CHECK(cudaMalloc(&d_A_ptrs_tiny, batch * sizeof(float *)));
+ CUDA_CHECK(cudaMalloc(&d_B_ptrs_tiny, batch * sizeof(float *)));
+
+ int threads = (batch + 255) / 256 * 256;
+ if (threads > 256) threads = 256;
+ int blocks = (batch + threads - 1) / threads;
+
+ setup_batch_pointers<<<blocks, threads>>>(d_A_ptrs_cublas, d_B_ptrs_cublas,
+ d_A, d_B_cublas, A_size, B_size, batch);
+ setup_batch_pointers<<<blocks, threads>>>(d_A_ptrs_tiny, d_B_ptrs_tiny,
+ d_A, d_B_tinyblas, A_size, B_size, batch);
+ CUDA_CHECK(cudaDeviceSynchronize());
+
+ // ---- Create handles ----
+ cublasHandle_t cublas_handle;
+ tinyblasHandle_t tinyblas_handle;
+ CUBLAS_CHECK(cublasCreate(&cublas_handle));
+ TINYBLAS_CHECK(tinyblasCreate(&tinyblas_handle));
+
+ // Use default math mode for cuBLAS (not TF32) to get accurate reference
+ CUBLAS_CHECK(cublasSetMathMode(cublas_handle, CUBLAS_DEFAULT_MATH));
+
+ // Set stream on tinyblas handle (tinyblasCreate uses malloc, not calloc,
+ // so the stream field is uninitialized)
+ TINYBLAS_CHECK(tinyblasSetStream(tinyblas_handle, nullptr));
+
+ // ---- Correctness run ----
+ CUBLAS_CHECK(cublasStrsmBatched(cublas_handle, CUBLAS_SIDE_RIGHT,
+ CUBLAS_FILL_MODE_UPPER, CUBLAS_OP_N,
+ CUBLAS_DIAG_NON_UNIT, m, n, &alpha,
+ d_A_ptrs_cublas, lda,
+ d_B_ptrs_cublas, ldb, batch));
+ CUDA_CHECK(cudaDeviceSynchronize());
+
+ TINYBLAS_CHECK(tinyblasStrsmBatched(tinyblas_handle, TINYBLAS_SIDE_RIGHT,
+ TINYBLAS_FILL_MODE_UPPER, TINYBLAS_OP_N,
+ TINYBLAS_DIAG_NON_UNIT, m, n, &alpha,
+ (const float *const *)d_A_ptrs_tiny, lda,
+ (float *const *)d_B_ptrs_tiny, ldb, batch));
+ CUDA_CHECK(cudaDeviceSynchronize());
+
+ // ---- Copy results back ----
+ std::vector<float> h_result_cublas(B_total);
+ std::vector<float> h_result_tinyblas(B_total);
+ CUDA_CHECK(cudaMemcpy(h_result_cublas.data(), d_B_cublas,
+ B_total * sizeof(float), cudaMemcpyDeviceToHost));
+ CUDA_CHECK(cudaMemcpy(h_result_tinyblas.data(), d_B_tinyblas,
+ B_total * sizeof(float), cudaMemcpyDeviceToHost));
+
+ // ---- Compare results ----
+ double sum_sq = 0.0;
+ double max_abs = 0.0;
+ long long max_ulp = 0;
+ int total_elements = 0;
+
+ for (int b = 0; b < batch; b++) {
+ for (int j = 0; j < n; j++) {
+ for (int i = 0; i < m; i++) {
+ float c = h_result_cublas[b * B_size + j * ldb + i];
+ float t = h_result_tinyblas[b * B_size + j * ldb + i];
+
+ if (std::isnan(c) || std::isnan(t)) {
+ fprintf(stderr, " NaN detected: batch=%d i=%d j=%d cublas=%g tinyblas=%g\n",
+ b, i, j, c, t);
+ result.pass = false;
+ goto cleanup;
+ }
+ if (std::isinf(c) || std::isinf(t)) {
+ fprintf(stderr, " Inf detected: batch=%d i=%d j=%d cublas=%g tinyblas=%g\n",
+ b, i, j, c, t);
+ result.pass = false;
+ goto cleanup;
+ }
+
+ double err = fabs((double)c - (double)t);
+ sum_sq += err * err;
+ if (err > max_abs)
+ max_abs = err;
+
+ long long ulp = ulp_diff(c, t);
+ if (ulp > max_ulp)
+ max_ulp = ulp;
+
+ total_elements++;
+ }
+ }
+ }
+
+ result.max_abs_err = max_abs;
+ result.rmse = (total_elements > 0) ? sqrt(sum_sq / total_elements) : 0.0;
+ result.max_ulp = max_ulp;
+
+ // ---- Residual: ||X*A - alpha*B|| / (||A||*||X||*n*eps) ----
+ // Computed per batch element, report the worst
+ {
+ double worst_cublas_res = 0.0, worst_tiny_res = 0.0;
+ for (int b = 0; b < batch; b++) {
+ double cr = compute_residual(
+ h_result_cublas.data() + b * B_size,
+ h_A.data() + b * A_size,
+ h_B.data() + b * B_size,
+ m, n, lda, ldb, alpha);
+ double tr = compute_residual(
+ h_result_tinyblas.data() + b * B_size,
+ h_A.data() + b * A_size,
+ h_B.data() + b * B_size,
+ m, n, lda, ldb, alpha);
+ if (cr > worst_cublas_res) worst_cublas_res = cr;
+ if (tr > worst_tiny_res) worst_tiny_res = tr;
+ }
+ result.cublas_residual = worst_cublas_res;
+ result.tinyblas_residual = worst_tiny_res;
+ }
+
+ // ---- Timing: cuBLAS ----
+ {
+ // Warmup
+ CUDA_CHECK(cudaMemcpy(d_B_cublas, h_B.data(), B_total * sizeof(float), cudaMemcpyHostToDevice));
+ CUBLAS_CHECK(cublasStrsmBatched(cublas_handle, CUBLAS_SIDE_RIGHT,
+ CUBLAS_FILL_MODE_UPPER, CUBLAS_OP_N,
+ CUBLAS_DIAG_NON_UNIT, m, n, &alpha,
+ d_A_ptrs_cublas, lda,
+ d_B_ptrs_cublas, ldb, batch));
+ CUDA_CHECK(cudaDeviceSynchronize());
+
+ long long total_us = 0;
+ for (int iter = 0; iter < cfg.iterations; iter++) {
+ // Reset B for each iteration (STRSM is in-place)
+ CUDA_CHECK(cudaMemcpy(d_B_cublas, h_B.data(), B_total * sizeof(float), cudaMemcpyHostToDevice));
+ CUDA_CHECK(cudaDeviceSynchronize());
+
+ long long t0 = micros();
+ CUBLAS_CHECK(cublasStrsmBatched(cublas_handle, CUBLAS_SIDE_RIGHT,
+ CUBLAS_FILL_MODE_UPPER, CUBLAS_OP_N,
+ CUBLAS_DIAG_NON_UNIT, m, n, &alpha,
+ d_A_ptrs_cublas, lda,
+ d_B_ptrs_cublas, ldb, batch));
+ CUDA_CHECK(cudaDeviceSynchronize());
+ total_us += micros() - t0;
+ }
+ result.cublas_us = (double)total_us / cfg.iterations;
+ }
+
+ // ---- Timing: TinyBLAS ----
+ {
+ // Warmup
+ CUDA_CHECK(cudaMemcpy(d_B_tinyblas, h_B.data(), B_total * sizeof(float), cudaMemcpyHostToDevice));
+ TINYBLAS_CHECK(tinyblasStrsmBatched(tinyblas_handle, TINYBLAS_SIDE_RIGHT,
+ TINYBLAS_FILL_MODE_UPPER, TINYBLAS_OP_N,
+ TINYBLAS_DIAG_NON_UNIT, m, n, &alpha,
+ (const float *const *)d_A_ptrs_tiny, lda,
+ (float *const *)d_B_ptrs_tiny, ldb, batch));
+ CUDA_CHECK(cudaDeviceSynchronize());
+
+ long long total_us = 0;
+ for (int iter = 0; iter < cfg.iterations; iter++) {
+ CUDA_CHECK(cudaMemcpy(d_B_tinyblas, h_B.data(), B_total * sizeof(float), cudaMemcpyHostToDevice));
+ CUDA_CHECK(cudaDeviceSynchronize());
+
+ long long t0 = micros();
+ TINYBLAS_CHECK(tinyblasStrsmBatched(tinyblas_handle, TINYBLAS_SIDE_RIGHT,
+ TINYBLAS_FILL_MODE_UPPER, TINYBLAS_OP_N,
+ TINYBLAS_DIAG_NON_UNIT, m, n, &alpha,
+ (const float *const *)d_A_ptrs_tiny, lda,
+ (float *const *)d_B_ptrs_tiny, ldb, batch));
+ CUDA_CHECK(cudaDeviceSynchronize());
+ total_us += micros() - t0;
+ }
+ result.tinyblas_us = (double)total_us / cfg.iterations;
+ }
+
+ result.pass = true;
+
+cleanup:
+ // ---- Cleanup ----
+ cublasDestroy(cublas_handle);
+ tinyblasDestroy(tinyblas_handle);
+ cudaFree(d_A);
+ cudaFree(d_B_cublas);
+ cudaFree(d_B_tinyblas);
+ cudaFree((void *)d_A_ptrs_cublas);
+ cudaFree(d_B_ptrs_cublas);
+ cudaFree((void *)d_A_ptrs_tiny);
+ cudaFree(d_B_ptrs_tiny);
+
+ return result;
+}
+
+// ============================================================================
+// Main
+// ============================================================================
+
+int main(int argc, char *argv[]) {
+ // Print GPU info
+ int device;
+ cudaDeviceProp prop;
+ CUDA_CHECK(cudaGetDevice(&device));
+ CUDA_CHECK(cudaGetDeviceProperties(&prop, device));
+
+ printf("============================================================================\n");
+ printf("STRSM Batched Test: TinyBLAS vs cuBLAS\n");
+ printf("============================================================================\n");
+ printf("GPU: %s (SM %d.%d)\n", prop.name, prop.major, prop.minor);
+ printf("Operation: X * A = alpha * B (SIDE_RIGHT, UPPER, OP_N, NON_UNIT)\n");
+ printf("\n");
+
+ // Define test cases
+ // Note: the fast CUDA kernel in solve_tri.cu handles n<=64 && k<=32,
+ // while larger sizes go through cublasStrsmBatched/tinyblasStrsmBatched.
+ // We test both ranges since the tinyblas kernel is the one being validated.
+
+ TestConfig tests[] = {
+ // =====================================================================
+ // Qwen3.5 / delta-net realistic cases (the actual production workload)
+ // Matrix A = I + upper_tri(K^T*K_b*decay), diagonal = 1.0
+ // Always n=64 (CHUNK_SIZE), m=64, batched over heads*chunks
+ // =====================================================================
+ {"qwen 64x64 b=1", 64, 64, 1, 1.0f, false, false, true, 200},
+ {"qwen 64x64 b=4", 64, 64, 4, 1.0f, false, false, true, 200},
+ {"qwen 64x64 b=32", 64, 64, 32, 1.0f, false, false, true, 100},
+ {"qwen 64x64 b=128", 64, 64, 128, 1.0f, false, false, true, 50},
+ {"qwen 64x64 b=256", 64, 64, 256, 1.0f, false, false, true, 20},
+ {"qwen 64x64 b=512", 64, 64, 512, 1.0f, false, false, true, 10},
+ {"qwen 64x64 b=1024", 64, 64, 1024, 1.0f, false, false, true, 5},
+ {"qwen 64x64 b=2048", 64, 64, 2048, 1.0f, false, false, true, 2},
+
+ // =====================================================================
+ // Corner cases with random matrices
+ // =====================================================================
+
+ // Basic small sizes
+ {"minimal 1x1", 1, 1, 1, 1.0f, false, false, false, 200},
+ {"small 2x2", 2, 2, 1, 1.0f, false, false, false, 200},
+ {"small 4x4", 4, 4, 1, 1.0f, false, false, false, 200},
+ {"small 8x8", 8, 8, 1, 1.0f, false, false, false, 200},
+
+ // Identity matrix (result should be alpha * B)
+ {"identity 8x8", 8, 8, 1, 1.0f, true, false, false, 200},
+ {"identity 64x32", 64, 32, 4, 1.0f, true, false, false, 200},
+
+ // Warp boundary cases (warp size = 32)
+ {"warp-exact 32x32", 32, 32, 1, 1.0f, false, false, false, 200},
+ {"warp+1 33x33", 33, 33, 1, 1.0f, false, false, false, 200},
+ {"warp-1 31x31", 31, 31, 1, 1.0f, false, false, false, 200},
+
+ // At/above the fast-kernel threshold (n=64, k=32 in solve_tri.cu)
+ {"threshold 64x32", 64, 32, 4, 1.0f, false, false, false, 100},
+ {"large 65x33", 65, 33, 4, 1.0f, false, false, false, 100},
+ {"large 128x64", 128, 64, 4, 1.0f, false, false, false, 50},
+
+ // Alpha scaling
+ {"alpha=0.5", 64, 32, 4, 0.5f, false, false, false, 100},
+ {"alpha=2.0", 64, 32, 4, 2.0f, false, false, false, 100},
+
+ // Rectangular
+ {"rect wide-A", 128, 8, 4, 1.0f, false, false, false, 100},
+ {"rect many-rows", 16, 512, 4, 1.0f, false, false, false, 50},
+
+ // Many batches
+ {"many-batch", 64, 32, 64, 1.0f, false, false, false, 50},
+
+ // Ill-conditioned (diagonal spans 1e-3 to 1e3)
+ {"ill-conditioned", 64, 32, 4, 1.0f, false, true, false, 100},
+ };
+
+ int num_tests = sizeof(tests) / sizeof(tests[0]);
+ int pass_count = 0;
+ int fail_count = 0;
+ double worst_max_abs = 0.0;
+ double worst_rmse = 0.0;
+ long long worst_ulp = 0;
+ double worst_cublas_res = 0.0;
+ double worst_tinyblas_res = 0.0;
+
+ for (int t = 0; t < num_tests; t++) {
+ const TestConfig &cfg = tests[t];
+
+ // Reset RNG for reproducibility per test
+ rand_state = 42 + t * 1000;
+
+ printf("Test: %-22s n=%-4d m=%-4d batch=%-3d alpha=%.1f\n",
+ cfg.name, cfg.n, cfg.m, cfg.batch, cfg.alpha);
+
+ TestResult r = run_test(cfg);
+
+ if (!r.pass) {
+ printf(" FAIL (NaN/Inf detected)\n\n");
+ fail_count++;
+ continue;
+ }
+
+ double speedup = (r.tinyblas_us > 0) ? r.cublas_us / r.tinyblas_us : 0.0;
+
+ printf(" cuBLAS: %8.1f us\n", r.cublas_us);
+ printf(" TinyBLAS: %8.1f us\n", r.tinyblas_us);
+ printf(" Speedup: %8.2fx\n", speedup);
+ printf(" Max abs error: %.6e (cublas vs tinyblas)\n", r.max_abs_err);
+ printf(" RMSE: %.6e\n", r.rmse);
+ printf(" Max ULP diff: %lld\n", r.max_ulp);
+ printf(" cuBLAS residual: %.2f (||XA-aB||/||A||||X||n*eps, <1 = good)\n", r.cublas_residual);
+ printf(" TinyBLAS resid: %.2f\n", r.tinyblas_residual);
+
+ // A scaled residual < ~10 means the implementation is numerically sound.
+ // Both cublas and tinyblas should achieve this for well-conditioned problems.
+ const double RESIDUAL_TOL = 10.0;
+ bool cublas_ok = r.cublas_residual < RESIDUAL_TOL;
+ bool tinyblas_ok = r.tinyblas_residual < RESIDUAL_TOL;
+
+ const char *verdict;
+ if (cublas_ok && tinyblas_ok) {
+ verdict = "PASS";
+ pass_count++;
+ } else if (!tinyblas_ok && cublas_ok) {
+ verdict = "FAIL (tinyblas residual too large)";
+ fail_count++;
+ } else if (!cublas_ok && !tinyblas_ok) {
+ verdict = "SKIP (both residuals large - ill-conditioned)";
+ pass_count++; // Don't count ill-conditioned failures
+ } else {
+ verdict = "FAIL (unexpected: cublas worse than tinyblas)";
+ fail_count++;
+ }
+ printf(" %s\n\n", verdict);
+
+ if (r.max_abs_err > worst_max_abs) worst_max_abs = r.max_abs_err;
+ if (r.rmse > worst_rmse) worst_rmse = r.rmse;
+ if (r.max_ulp > worst_ulp) worst_ulp = r.max_ulp;
+ if (r.cublas_residual > worst_cublas_res) worst_cublas_res = r.cublas_residual;
+ if (r.tinyblas_residual > worst_tinyblas_res) worst_tinyblas_res = r.tinyblas_residual;
+ }
+
+ // ---- Summary ----
+ printf("============================================================================\n");
+ printf("Summary: %d/%d passed", pass_count, num_tests);
+ if (fail_count > 0)
+ printf(", %d FAILED", fail_count);
+ printf("\n");
+ printf("Worst max abs error: %.6e\n", worst_max_abs);
+ printf("Worst RMSE: %.6e\n", worst_rmse);
+ printf("Worst max ULP diff: %lld\n", worst_ulp);
+ printf("Worst cuBLAS residual: %.2f\n", worst_cublas_res);
+ printf("Worst TinyBLAS resid: %.2f\n", worst_tinyblas_res);
+ printf("============================================================================\n");
+
+ return fail_count > 0 ? 1 : 0;
+}