Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
//
// Copyright 2026 Mozilla Foundation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Numerical-equivalence tests for the llamafile flash-attention helpers
// (issue #975).
//
// These tests compare our AVX-512-optimized llamafile_fa_* helpers
// against the upstream ggml-cpu reference implementations. Goal: ensure
// the alternative low-level math we ship via llamafile/sgemm.cpp's
// dispatch produces output that's numerically equivalent to the
// reference (bit-identical for the conversion helper, within ULP
// tolerance for the reduction helper).
//
// Run via `make check`. On CPUs without AVX-512F the helpers report
// "not supported" and the corresponding assertions are skipped (the
// test reports skipped count); the test still validates the dispatch
// table wiring.
#include "ggml.h"
#include "sgemm.h"
#include <cstdio>
#include <cstdint>
#include <cstring>
#include <cmath>
#include <cfloat>
#include <random>
#include <vector>
#include <algorithm>
// ggml_vec_dot_f16 is defined in ggml-cpu/vec.cpp but not declared in a
// public header (it's reached through type_traits dispatch). Declare it
// here for the test.
extern "C" void ggml_vec_dot_f16(int n, float *s, size_t bs,
ggml_fp16_t *x, size_t bx,
ggml_fp16_t *y, size_t by, int nrc);
static int test_count = 0;
static int fail_count = 0;
static int skip_count = 0;
// Each TEST() body runs from main(), NOT at static-init time, because
// the upstream ggml helpers we compare against (ggml_fp16_to_fp32_row
// in particular) depend on a lookup table populated by ggml's graph-
// compute first-call init. We init that via a tiny graph-compute in
// main() before invoking any test. Tests register themselves into a
// list at static-init time, then main() iterates and runs them.
struct TestEntry {
const char *name;
void (*fn)();
};
static TestEntry g_tests[64];
static int g_test_count_registered = 0;
#define TEST(name) \
static void test_##name(); \
static struct TestRegister_##name { \
TestRegister_##name() { \
g_tests[g_test_count_registered++] = { #name, test_##name }; \
} \
} test_register_##name; \
static void test_##name()
#define ASSERT_TRUE(cond, msg) \
do { \
test_count++; \
if (!(cond)) { \
fprintf(stderr, "FAIL: %s (%s:%d)\n condition: %s\n", \
msg, __FILE__, __LINE__, #cond); \
fail_count++; \
} \
} while (0)
#define ASSERT_BITEQ_F32(expected, actual, msg) \
do { \
test_count++; \
uint32_t e_bits, a_bits; \
memcpy(&e_bits, &(expected), sizeof(uint32_t)); \
memcpy(&a_bits, &(actual), sizeof(uint32_t)); \
if (e_bits != a_bits) { \
fprintf(stderr, "FAIL: %s (%s:%d)\n" \
" expected: %.9g (0x%08x)\n" \
" actual: %.9g (0x%08x)\n", \
msg, __FILE__, __LINE__, \
(double)(expected), e_bits, \
(double)(actual), a_bits); \
fail_count++; \
} \
} while (0)
#define ASSERT_NEAR_F32(expected, actual, tol, msg) \
do { \
test_count++; \
float diff = fabsf((expected) - (actual)); \
if (!(diff <= (tol))) { /* using <= to fail on NaN */ \
fprintf(stderr, "FAIL: %s (%s:%d)\n" \
" expected: %.9g\n" \
" actual: %.9g\n" \
" diff: %.9g (tol %.9g)\n", \
msg, __FILE__, __LINE__, \
(double)(expected), (double)(actual), \
(double)diff, (double)(tol)); \
fail_count++; \
} \
} while (0)
// Test sizes covering typical attention head dims plus odd tail sizes
// to exercise the scalar leftover loop.
static const std::vector<int> kTestSizes = {
1, 8, 15, 16, 17, 32, 33, 64, 80, 96, 128, 129, 192, 256, 512, 1024
};
// Fill a vector with random f16 values in N(0, 0.1) — typical layer
// activation range after LayerNorm. Uses a fixed seed for determinism.
static void fill_random_f16(std::vector<ggml_fp16_t> &out, uint64_t seed) {
std::mt19937_64 rng(seed);
std::normal_distribution<float> dist(0.0f, 0.1f);
for (auto &v : out) {
v = ggml_fp32_to_fp16(dist(rng));
}
}
// Tolerance for vec_dot_f16: both implementations convert to f32 and
// sum, but in different chunk sizes (AVX2: 8-wide; AVX-512: 16-wide).
// The reduction order differs, so we allow a relative error
// proportional to n (Wilkinson-style summed-product bound: O(n) terms
// each at unit-roundoff eps/2, with a small safety factor). max_abs
// here is the peak |x[i]*y[i]|, an upper bound for the running partial
// sum scale.
static float vec_dot_tol(int n, const std::vector<ggml_fp16_t> &x,
const std::vector<ggml_fp16_t> &y) {
float max_abs = 0.0f;
for (int i = 0; i < n; ++i) {
float xv = ggml_fp16_to_fp32(x[i]);
float yv = ggml_fp16_to_fp32(y[i]);
max_abs = std::max(max_abs, std::fabs(xv) * std::fabs(yv));
}
return std::max(1e-5f, 8.0f * (float)n * FLT_EPSILON * max_abs);
}
// ============================================================================
// fp16_to_fp32_row: per-element conversion. Both ggml's helper and ours
// use _mm*_cvtph_ps, which is IEEE-754 deterministic per element. We
// expect BIT-IDENTICAL output regardless of SIMD width.
// ============================================================================
TEST(fp16_to_fp32_row_normal_inputs) {
for (int n : kTestSizes) {
std::vector<ggml_fp16_t> input(n);
std::vector<float> ref_out(n), our_out(n);
fill_random_f16(input, /*seed=*/0xDEADBEEFCAFEull ^ (uint64_t)n);
ggml_fp16_to_fp32_row(input.data(), ref_out.data(), n);
bool handled = llamafile_fa_fp16_to_fp32_row(input.data(), our_out.data(), n);
if (!handled) {
skip_count++;
continue;
}
char msg[128];
for (int i = 0; i < n; ++i) {
snprintf(msg, sizeof(msg),
"fp16_to_fp32_row n=%d i=%d (normal inputs)", n, i);
ASSERT_BITEQ_F32(ref_out[i], our_out[i], msg);
}
}
}
TEST(fp16_to_fp32_row_edge_values) {
// Include ±0, ±INF, denormals, NaN, and the f16 min/max.
std::vector<ggml_fp16_t> input = {
ggml_fp32_to_fp16( 0.0f),
ggml_fp32_to_fp16(-0.0f),
ggml_fp32_to_fp16( INFINITY),
ggml_fp32_to_fp16(-INFINITY),
ggml_fp32_to_fp16(NAN),
ggml_fp32_to_fp16( 65504.0f), // f16 max finite
ggml_fp32_to_fp16(-65504.0f), // f16 min finite
ggml_fp32_to_fp16( 6.10e-5f), // f16 smallest normal
ggml_fp32_to_fp16( 5.96e-8f), // f16 smallest denormal
ggml_fp32_to_fp16( 1.0f),
ggml_fp32_to_fp16(-1.0f),
ggml_fp32_to_fp16( 0.5f),
ggml_fp32_to_fp16( 0.125f),
ggml_fp32_to_fp16( 0.0625f),
ggml_fp32_to_fp16( 0.00390625f),
ggml_fp32_to_fp16( 65000.0f),
};
int n = (int)input.size();
std::vector<float> ref_out(n), our_out(n);
ggml_fp16_to_fp32_row(input.data(), ref_out.data(), n);
bool handled = llamafile_fa_fp16_to_fp32_row(input.data(), our_out.data(), n);
if (!handled) {
skip_count++;
return;
}
char msg[128];
for (int i = 0; i < n; ++i) {
// For NaN, only assert both are NaN (any NaN representation is OK).
if (std::isnan(ref_out[i]) || std::isnan(our_out[i])) {
test_count++;
if (!(std::isnan(ref_out[i]) && std::isnan(our_out[i]))) {
fprintf(stderr,
"FAIL: fp16_to_fp32_row edge values i=%d: "
"NaN mismatch (ref nan=%d, our nan=%d)\n",
i, std::isnan(ref_out[i]), std::isnan(our_out[i]));
fail_count++;
}
continue;
}
snprintf(msg, sizeof(msg),
"fp16_to_fp32_row edge values i=%d", i);
ASSERT_BITEQ_F32(ref_out[i], our_out[i], msg);
}
}
// ============================================================================
// vec_dot_f16: reduction, allow ULP tolerance because chunk widths
// differ between ref (AVX2 8-wide) and ours (AVX-512 16-wide).
// ============================================================================
TEST(vec_dot_f16_normal_inputs) {
for (int n : kTestSizes) {
std::vector<ggml_fp16_t> x(n), y(n);
fill_random_f16(x, /*seed=*/0xA5A5A5A5A5A5A5A5ull ^ (uint64_t)n);
fill_random_f16(y, /*seed=*/0x5A5A5A5A5A5A5A5Aull ^ (uint64_t)n);
float ref = 0.0f, our = 0.0f;
ggml_vec_dot_f16(n, &ref, /*bs=*/0, x.data(), /*bx=*/0,
y.data(), /*by=*/0, /*nrc=*/1);
bool handled = llamafile_fa_vec_dot_f16(n, &our, x.data(), y.data());
if (!handled) {
skip_count++;
continue;
}
char msg[128];
snprintf(msg, sizeof(msg),
"vec_dot_f16 n=%d (normal inputs)", n);
ASSERT_NEAR_F32(ref, our, vec_dot_tol(n, x, y), msg);
}
}
TEST(vec_dot_f16_adversarial_reduction_order) {
// To actually stress reduction-order sensitivity, the per-element
// products x[i]*y[i] must alternate sign — otherwise SIMD width
// doesn't change the result. Use a scale of 10 so |product| = 100
// (well within f16 range): partial sums oscillate by ±100 across
// steps, and AVX2 (8-wide) vs AVX-512 (16-wide) reduction trees
// pair the cancelling terms in different orders, amplifying any
// floating-point drift.
// x[i] = (i & 1) ? -10 : +10 → [+, -, +, -, ...]
// y[i] = ((i / 2) & 1) ? -10 : +10 → [+, +, -, -, ...]
// products = x[i]*y[i] alternate as [+, -, -, +, +, -, -, +, ...]
for (int n : kTestSizes) {
if (n < 2) continue;
std::vector<ggml_fp16_t> x(n), y(n);
const float scale = 10.0f;
for (int i = 0; i < n; ++i) {
x[i] = ggml_fp32_to_fp16((i & 1) ? -scale : scale);
y[i] = ggml_fp32_to_fp16(((i / 2) & 1) ? -scale : scale);
}
float ref = 0.0f, our = 0.0f;
ggml_vec_dot_f16(n, &ref, 0, x.data(), 0, y.data(), 0, 1);
bool handled = llamafile_fa_vec_dot_f16(n, &our, x.data(), y.data());
if (!handled) {
skip_count++;
continue;
}
// Worst-case drift bound: ~n * eps * max_partial_sum_scale.
// max |x[i]*y[i]| = scale^2 = 100. Use 4 * n * eps * 100 with
// a small absolute floor for tiny n.
float tol = std::max(1e-4f, 4.0f * (float)n * FLT_EPSILON * scale * scale);
char msg[128];
snprintf(msg, sizeof(msg),
"vec_dot_f16 n=%d (adversarial alternating-sign products)", n);
ASSERT_NEAR_F32(ref, our, tol, msg);
}
}
TEST(vec_dot_f16_zeros) {
// All-zero inputs: result must be exactly 0.0f, bit-identical.
for (int n : kTestSizes) {
std::vector<ggml_fp16_t> x(n, ggml_fp32_to_fp16(0.0f));
std::vector<ggml_fp16_t> y(n, ggml_fp32_to_fp16(0.0f));
float ref = 999.0f, our = 999.0f;
ggml_vec_dot_f16(n, &ref, 0, x.data(), 0, y.data(), 0, 1);
bool handled = llamafile_fa_vec_dot_f16(n, &our, x.data(), y.data());
if (!handled) {
skip_count++;
continue;
}
float zero = 0.0f;
char msg[128];
snprintf(msg, sizeof(msg), "vec_dot_f16 n=%d (zeros) ref", n);
ASSERT_BITEQ_F32(zero, ref, msg);
snprintf(msg, sizeof(msg), "vec_dot_f16 n=%d (zeros) ours", n);
ASSERT_BITEQ_F32(zero, our, msg);
}
}
TEST(dispatch_wiring) {
// Sanity check: regardless of whether the AVX-512F variant is
// available, the wrapper must return without crashing for trivial
// input. If our dispatch table got mis-wired (e.g., function pointer
// to wrong arch), this would catch it.
std::vector<ggml_fp16_t> tiny = {
ggml_fp32_to_fp16(0.5f), ggml_fp32_to_fp16(-0.25f),
ggml_fp32_to_fp16(1.0f), ggml_fp32_to_fp16( 0.125f),
};
float dot = -1.0f;
bool dot_handled = llamafile_fa_vec_dot_f16((int)tiny.size(), &dot,
tiny.data(), tiny.data());
ASSERT_TRUE(dot_handled || dot == -1.0f,
"vec_dot_f16: if not handled, output must be untouched");
std::vector<float> out(tiny.size(), -999.0f);
bool conv_handled = llamafile_fa_fp16_to_fp32_row(tiny.data(), out.data(),
(int64_t)tiny.size());
ASSERT_TRUE(conv_handled || out[0] == -999.0f,
"fp16_to_fp32_row: if not handled, output must be untouched");
}
// ============================================================================
// simd_gemm: C[M x N] += A[M x K] * B[K x N], all f32. We compare our
// AVX-512 wrapper against a scalar reference because the upstream
// ggml-cpu simd_gemm is `static inline` in a header and isn't directly
// linkable here. The scalar reference is the exact math the wrapper
// must reproduce; both implementations are well-defined up to FMA
// associativity (FMA rounding doesn't commute across reduction orders).
// Tolerance: 4 * K * eps * max(|A|*|B|) which matches the worst-case
// summed-product bound.
// ============================================================================
static void scalar_gemm_reference(float *C, const float *A, const float *B,
int M, int K, int N) {
for (int i = 0; i < M; ++i) {
for (int j = 0; j < N; ++j) {
float acc = C[i * N + j];
for (int k = 0; k < K; ++k) {
acc += A[i * K + k] * B[k * N + j];
}
C[i * N + j] = acc;
}
}
}
static float gemm_tol(int K, float scale) {
return std::max(1e-4f, 8.0f * (float)K * FLT_EPSILON * scale * scale);
}
static void fill_random_f32(std::vector<float> &v, uint64_t seed,
float stddev = 0.1f) {
std::mt19937_64 rng(seed);
std::normal_distribution<float> dist(0.0f, stddev);
for (auto &x : v) x = dist(rng);
}
TEST(simd_gemm_typical_fa_tile_shapes) {
// Shapes that match the FA tile dispatch in ops.cpp's tiled function
// (Q_TILE_SZ x KV_TILE_SZ for the KQ matmul, Q_TILE_SZ x DV for the
// VKQ accumulation). Common head dims: 64, 80, 96, 128. Common tile
// sizes from ggml_fa_tile_config: 4-16 Q rows, 32-128 KV columns.
struct Shape { int M, K, N; const char *label; };
Shape shapes[] = {
{ 4, 128, 64, "KQ: Q4 x DK128 x KV64" },
{ 8, 128, 64, "KQ: Q8 x DK128 x KV64" },
{ 16, 128, 128, "KQ: Q16 x DK128 x KV128" },
{ 4, 64, 128, "VKQ: Q4 x KV64 x DV128" },
{ 8, 64, 128, "VKQ: Q8 x KV64 x DV128" },
{ 16, 128, 128, "VKQ: Q16 x KV128 x DV128" },
{ 4, 96, 80, "odd: Q4 x 96 x 80" },
{ 16, 64, 32, "narrow: Q16 x 64 x 32" },
};
for (auto &s : shapes) {
std::vector<float> A(s.M * s.K), B(s.K * s.N);
std::vector<float> C_ref(s.M * s.N), C_our(s.M * s.N);
uint64_t seed = 0xC0FFEEull ^ (uint64_t)(s.M * 31 + s.K * 17 + s.N);
fill_random_f32(A, seed);
fill_random_f32(B, seed ^ 0xDEADBEEFull);
// Pre-fill C with random values to test the += accumulation.
fill_random_f32(C_ref, seed ^ 0xFACEFEEDull);
C_our = C_ref;
scalar_gemm_reference(C_ref.data(), A.data(), B.data(),
s.M, s.K, s.N);
bool handled = llamafile_fa_simd_gemm(C_our.data(), A.data(),
B.data(), s.M, s.K, s.N);
if (!handled) {
skip_count++;
continue;
}
char msg[160];
float tol = gemm_tol(s.K, 0.5f); // stddev 0.1 → max |a|*|b| ~ 0.5
for (int i = 0; i < s.M; ++i) {
for (int j = 0; j < s.N; ++j) {
snprintf(msg, sizeof(msg),
"simd_gemm %s i=%d j=%d", s.label, i, j);
ASSERT_NEAR_F32(C_ref[i * s.N + j], C_our[i * s.N + j], tol, msg);
}
}
}
}
TEST(simd_gemm_zero_accumulator) {
// Start from C=0 so the result is purely A*B (no += round-off).
// Verifies the kernel handles a zero starting accumulator correctly.
int M = 8, K = 128, N = 128;
std::vector<float> A(M * K), B(K * N);
std::vector<float> C_ref(M * N, 0.0f), C_our(M * N, 0.0f);
fill_random_f32(A, 0x1234);
fill_random_f32(B, 0x5678);
scalar_gemm_reference(C_ref.data(), A.data(), B.data(), M, K, N);
bool handled = llamafile_fa_simd_gemm(C_our.data(), A.data(), B.data(),
M, K, N);
if (!handled) {
skip_count++;
return;
}
char msg[160];
float tol = gemm_tol(K, 0.5f);
for (int i = 0; i < M; ++i) {
for (int j = 0; j < N; ++j) {
snprintf(msg, sizeof(msg),
"simd_gemm zero-acc i=%d j=%d", i, j);
ASSERT_NEAR_F32(C_ref[i * N + j], C_our[i * N + j], tol, msg);
}
}
}
TEST(simd_gemm_dispatch_wiring) {
int M = 4, K = 16, N = 32;
std::vector<float> A(M * K, 1.0f), B(K * N, 1.0f);
std::vector<float> C(M * N, -777.0f);
bool handled = llamafile_fa_simd_gemm(C.data(), A.data(), B.data(),
M, K, N);
ASSERT_TRUE(handled || C[0] == -777.0f,
"simd_gemm: if not handled, output must be untouched");
if (handled) {
// Each C[i,j] = -777 + sum_k(1 * 1) = -777 + K = -761
for (int i = 0; i < M; ++i) {
for (int j = 0; j < N; ++j) {
test_count++;
float expected = -777.0f + (float)K;
if (fabsf(C[i * N + j] - expected) > 1e-5f) {
fprintf(stderr, "FAIL: simd_gemm dispatch i=%d j=%d "
"expected %.3f got %.3f\n",
i, j, expected, C[i * N + j]);
fail_count++;
}
}
}
}
}
// ggml's f16↔f32 lookup table (used by ggml_vec_dot_f16's scalar tail
// in simd-mappings.h via GGML_CPU_FP16_TO_FP32). Populated by ggml's
// graph compute first-call init; we replicate the population here so
// the tests don't depend on running a graph.
extern "C" float ggml_table_f32_f16[1 << 16];
static void init_ggml_tables() {
for (int i = 0; i < (1 << 16); ++i) {
ggml_fp16_t f16 = (ggml_fp16_t)(uint16_t)i;
ggml_table_f32_f16[i] = ggml_fp16_to_fp32(f16);
}
}
int main(int argc, char *argv[]) {
(void)argc;
(void)argv;
// Must happen before any test runs ggml_vec_dot_f16.
init_ggml_tables();
fprintf(stderr, "Running fa_helpers tests (issue #975) — %d tests registered\n",
g_test_count_registered);
for (int i = 0; i < g_test_count_registered; ++i) {
fprintf(stderr, " [%d/%d] %s\n", i + 1, g_test_count_registered,
g_tests[i].name);
g_tests[i].fn();
}
if (fail_count > 0) {
fprintf(stderr, "\n%d/%d assertions FAILED (%d skipped — CPU "
"doesn't have the optimized variant)\n",
fail_count, test_count, skip_count);
return 1;
}
fprintf(stderr, "All %d assertions PASSED (%d skipped — CPU doesn't "
"have the optimized variant)\n",
test_count, skip_count);
return 0;
}