37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
/*
* gpu_fma.c - whether the driver rounds fma() once, as the CPU does, and
* the shaders' _xfma variants where it does not. RADV splits every 32-bit
* fma into a multiply and an add on the AMD GPUs before RDNA2 (Polaris,
* Vega, the Renoir APUs, RDNA1: NIR's lower_ffma32, NoContraction or not),
* two roundings: products 1 ulp off the CPU's on half the rows, issue #48.
* When the GPU opens, fma_probe.comp computes fma() of triples where one
* rounding and two differ (c = -a*b rounded: once gives a*b's lost bits,
* twice gives 0); if the driver splits it, every shader that calls fma()
* is built from its variant (shaders/xfma.glsl: fma rounded once through
* float64), provided the variant passes the same probe. JANAS_GPU_XFMA=1
* takes the variants anyway, =0 never (both to test them).
*/
#include "gpu.h"
#if defined(JANAS_GPU_SPV) && __has_include(<vulkan/vulkan.h>)
#include "gpu_impl.h"
#include <math.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "fma_probe.h" /* janas_spv_fma_probe, generated by build.sh */
#include "xfma_table.h" /* janas_xfma_table, generated by build.sh */
enum { N_PROBE = 1024 };
const uint32_t *janas_g_spv(const struct janas_gpu *g, const uint32_t *code,
size_t *bytes)
{
if (g->xfma)
for (size_t i = 0;
i < sizeof(janas_xfma_table) / sizeof(*janas_xfma_table); i++)
if (janas_xfma_table[i].code == code) {
*bytes = janas_xfma_table[i].bytes;
return janas_xfma_table[i].xfma;
}
return code;
}
static uint32_t next(uint32_t *s)
{
*s ^= *s << 13;
*s ^= *s >> 17;
*s ^= *s << 5;
return *s;
}
/* A float of about 1 in magnitude, either sign, all 24 bits used. */
static float operand(uint32_t *s)
{
uint32_t r = next(s);
float m = 1.0f + (float)(r & 0x7fffff) / 8388608.0f;
return ldexpf(r >> 31 ? -m : m, (int)(r >> 23 & 7) - 3);
}
/* How many of the probe's triples the GPU's fma() gets wrong (against the
C library's, rounded once), -1 if it could not run. */
static int probe(struct janas_gpu *g)
{
const char *e = getenv("JANAS_GPU_FMA_N"); /* more triples, to look */
int np = e && atoi(e) > N_PROBE ? atoi(e) : N_PROBE;
size_t in_bytes = 3 * (size_t)np * sizeof(float),
bytes = in_bytes + (size_t)np * sizeof(float);
struct stage st = {0};
VkShaderModule sm = VK_NULL_HANDLE;
VkPipeline pipe = VK_NULL_HANDLE;
int wrong = -1;
if (janas_g_make_stage(g, &st, bytes, 0) != 0)
return -1;
if (janas_g_make_pipeline(g, janas_spv_fma_probe,
sizeof(janas_spv_fma_probe), &sm, &pipe,
0) != VK_SUCCESS)
goto done;
float *abc = (float *)st.map, *y = abc + 3 * (size_t)np;
uint32_t seed = 0x9e3779b9u;
for (int i = 0; i < np; i++) {
float a = operand(&seed), b = operand(&seed), c;
if (i % 4 == 0) /* c = -(a*b rounded): one rounding and two part */
c = -(a * b);
else if (i % 4 == 1) /* any c */
c = operand(&seed);
else if (i % 4 == 2) { /* a product's: scale, integer, sum */
a = ldexpf(a, -12);
b = (float)(int)(next(&seed) % 2000000) - 1e6f;
c = ldexpf(operand(&seed), 8);
} else /* a*b just off -c: the sum loses bits */
c = -(a * b) * (1.0f + ldexpf((float)(next(&seed) & 255), -22));
abc[3 * i] = a;
abc[3 * i + 1] = b;
abc[3 * i + 2] = c;
}
memset(y, 0, (size_t)np * sizeof(float));
VkCommandBufferBeginInfo bb = {
.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
if (g->vkResetCommandBuffer(g->cb, 0) != VK_SUCCESS ||
g->vkBeginCommandBuffer(g->cb, &bb) != VK_SUCCESS)
goto done;
g->vkCmdBindPipeline(g->cb, VK_PIPELINE_BIND_POINT_COMPUTE, pipe);
struct {
uint32_t abc[2], y[2];
uint32_t n;
} pc = {.n = (uint32_t)np};
janas_g_push_addr(pc.abc, st.addr);
janas_g_push_addr(pc.y, st.addr + in_bytes);
g->vkCmdPushConstants(g->cb, g->pl, VK_SHADER_STAGE_COMPUTE_BIT, 0,
sizeof(pc), &pc);
g->vkCmdDispatch(g->cb, (uint32_t)(np + 63) / 64, 1, 1);
VkMemoryBarrier mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT,
.dstAccessMask = VK_ACCESS_HOST_READ_BIT};
g->vkCmdPipelineBarrier(g->cb, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &mb, 0, NULL, 0,
NULL);
if (g->vkEndCommandBuffer(g->cb) != VK_SUCCESS)
goto done;
VkSubmitInfo sub = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
.commandBufferCount = 1,
.pCommandBuffers = &g->cb};
if (g->vkQueueSubmit(g->queue, 1, &sub, g->fence) != VK_SUCCESS)
goto done;
VkResult r =
g->vkWaitForFences(g->dev, 1, &g->fence, VK_TRUE, 5000000000ull);
g->vkResetFences(g->dev, 1, &g->fence);
if (r != VK_SUCCESS)
goto done;
wrong = 0;
for (int i = 0; i < np; i++) {
float want = fmaf(abc[3 * i], abc[3 * i + 1], abc[3 * i + 2]);
if (memcmp(&want, &y[i], sizeof(float)) != 0 && wrong++ < 4 && e)
fprintf(stderr, "gpu fma: fma(%a, %a, %a) = %a, the GPU's %a\n",
abc[3 * i], abc[3 * i + 1], abc[3 * i + 2], want, y[i]);
}
done:
if (pipe)
g->vkDestroyPipeline(g->dev, pipe, NULL);
if (sm)
g->vkDestroyShaderModule(g->dev, sm, NULL);
janas_g_free_stage(g, &st);
return wrong;
}
void janas_g_fma_probe(struct janas_gpu *g)
{
const char *e = getenv("JANAS_GPU_XFMA");
int forced = e && *e ? atoi(e) > 0 : -1;
g->xfma = 0;
int plain = forced < 0 ? probe(g) : -1;
int verbose = getenv("JANAS_GPU_VERBOSE") != NULL;
if (forced == 0 || (forced < 0 && plain <= 0)) {
if (verbose)
fprintf(stderr, "gpu fma: %s\n",
forced == 0 ? "the shaders' own (JANAS_GPU_XFMA=0)"
: plain == 0 ? "rounded once"
: "probe failed");
return;
}
int with = -1;
if (g->has_f64) {
g->xfma = 1;
with = probe(g);
if (with != 0 && forced < 0)
g->xfma = 0;
}
size_t n = strlen(g->about);
snprintf(g->about + n, sizeof(g->about) - n, "; fma %s",
forced > 0 ? (g->xfma ? "in float64 (JANAS_GPU_XFMA=1)"
: "the shaders' own (no float64)")
: g->xfma ? "split by the driver, rounded once in float64"
: g->has_f64 ? "split by the driver (float64 did not mend it)"
: "split by the driver (no float64)");
if (verbose || (forced < 0 && !g->xfma))
fprintf(stderr,
"gpu fma: %d of the probe's triples split by the driver; %s "
"(%d wrong with them)\n",
plain,
g->xfma ? "the shaders' float64 variants" : "the shaders' own",
with);
}
#endif