Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

7.2 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
/*
 * gpu_fma.c - whether the driver rounds fma() once, as the CPU does, and
 * the shaders' _xfma variants where it does not. RADV splits every 32-bit
 * fma into a multiply and an add on the AMD GPUs before RDNA2 (Polaris,
 * Vega, the Renoir APUs, RDNA1: NIR's lower_ffma32, NoContraction or not),
 * two roundings: products 1 ulp off the CPU's on half the rows, issue #48.
 * When the GPU opens, fma_probe.comp computes fma() of triples where one
 * rounding and two differ (c = -a*b rounded: once gives a*b's lost bits,
 * twice gives 0); if the driver splits it, every shader that calls fma()
 * is built from its variant (shaders/xfma.glsl: fma rounded once through
 * float64), provided the variant passes the same probe. JANAS_GPU_XFMA=1
 * takes the variants anyway, =0 never (both to test them).
 */
#include "gpu.h"

#if defined(JANAS_GPU_SPV) && __has_include(<vulkan/vulkan.h>)
#include "gpu_impl.h"

#include <math.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "fma_probe.h"  /* janas_spv_fma_probe, generated by build.sh */
#include "xfma_table.h" /* janas_xfma_table, generated by build.sh */

enum { N_PROBE = 1024 };

const uint32_t *janas_g_spv(const struct janas_gpu *g, const uint32_t *code,
                            size_t *bytes)
{
    if (g->xfma)
        for (size_t i = 0;
             i < sizeof(janas_xfma_table) / sizeof(*janas_xfma_table); i++)
            if (janas_xfma_table[i].code == code) {
                *bytes = janas_xfma_table[i].bytes;
                return janas_xfma_table[i].xfma;
            }
    return code;
}

static uint32_t next(uint32_t *s)
{
    *s ^= *s << 13;
    *s ^= *s >> 17;
    *s ^= *s << 5;
    return *s;
}

/* A float of about 1 in magnitude, either sign, all 24 bits used. */
static float operand(uint32_t *s)
{
    uint32_t r = next(s);
    float m = 1.0f + (float)(r & 0x7fffff) / 8388608.0f;
    return ldexpf(r >> 31 ? -m : m, (int)(r >> 23 & 7) - 3);
}

/* How many of the probe's triples the GPU's fma() gets wrong (against the
   C library's, rounded once), -1 if it could not run. */
static int probe(struct janas_gpu *g)
{
    const char *e = getenv("JANAS_GPU_FMA_N"); /* more triples, to look */
    int np = e && atoi(e) > N_PROBE ? atoi(e) : N_PROBE;
    size_t in_bytes = 3 * (size_t)np * sizeof(float),
           bytes = in_bytes + (size_t)np * sizeof(float);
    struct stage st = {0};
    VkShaderModule sm = VK_NULL_HANDLE;
    VkPipeline pipe = VK_NULL_HANDLE;
    int wrong = -1;
    if (janas_g_make_stage(g, &st, bytes, 0) != 0)
        return -1;
    if (janas_g_make_pipeline(g, janas_spv_fma_probe,
                              sizeof(janas_spv_fma_probe), &sm, &pipe,
                              0) != VK_SUCCESS)
        goto done;
    float *abc = (float *)st.map, *y = abc + 3 * (size_t)np;
    uint32_t seed = 0x9e3779b9u;
    for (int i = 0; i < np; i++) {
        float a = operand(&seed), b = operand(&seed), c;
        if (i % 4 == 0) /* c = -(a*b rounded): one rounding and two part */
            c = -(a * b);
        else if (i % 4 == 1) /* any c */
            c = operand(&seed);
        else if (i % 4 == 2) { /* a product's: scale, integer, sum */
            a = ldexpf(a, -12);
            b = (float)(int)(next(&seed) % 2000000) - 1e6f;
            c = ldexpf(operand(&seed), 8);
        } else /* a*b just off -c: the sum loses bits */
            c = -(a * b) * (1.0f + ldexpf((float)(next(&seed) & 255), -22));
        abc[3 * i] = a;
        abc[3 * i + 1] = b;
        abc[3 * i + 2] = c;
    }
    memset(y, 0, (size_t)np * sizeof(float));
    VkCommandBufferBeginInfo bb = {
        .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
        .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
    if (g->vkResetCommandBuffer(g->cb, 0) != VK_SUCCESS ||
        g->vkBeginCommandBuffer(g->cb, &bb) != VK_SUCCESS)
        goto done;
    g->vkCmdBindPipeline(g->cb, VK_PIPELINE_BIND_POINT_COMPUTE, pipe);
    struct {
        uint32_t abc[2], y[2];
        uint32_t n;
    } pc = {.n = (uint32_t)np};
    janas_g_push_addr(pc.abc, st.addr);
    janas_g_push_addr(pc.y, st.addr + in_bytes);
    g->vkCmdPushConstants(g->cb, g->pl, VK_SHADER_STAGE_COMPUTE_BIT, 0,
                          sizeof(pc), &pc);
    g->vkCmdDispatch(g->cb, (uint32_t)(np + 63) / 64, 1, 1);
    VkMemoryBarrier mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
                          .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT,
                          .dstAccessMask = VK_ACCESS_HOST_READ_BIT};
    g->vkCmdPipelineBarrier(g->cb, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
                            VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &mb, 0, NULL, 0,
                            NULL);
    if (g->vkEndCommandBuffer(g->cb) != VK_SUCCESS)
        goto done;
    VkSubmitInfo sub = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
                        .commandBufferCount = 1,
                        .pCommandBuffers = &g->cb};
    if (g->vkQueueSubmit(g->queue, 1, &sub, g->fence) != VK_SUCCESS)
        goto done;
    VkResult r =
        g->vkWaitForFences(g->dev, 1, &g->fence, VK_TRUE, 5000000000ull);
    g->vkResetFences(g->dev, 1, &g->fence);
    if (r != VK_SUCCESS)
        goto done;
    wrong = 0;
    for (int i = 0; i < np; i++) {
        float want = fmaf(abc[3 * i], abc[3 * i + 1], abc[3 * i + 2]);
        if (memcmp(&want, &y[i], sizeof(float)) != 0 && wrong++ < 4 && e)
            fprintf(stderr, "gpu fma: fma(%a, %a, %a) = %a, the GPU's %a\n",
                    abc[3 * i], abc[3 * i + 1], abc[3 * i + 2], want, y[i]);
    }
done:
    if (pipe)
        g->vkDestroyPipeline(g->dev, pipe, NULL);
    if (sm)
        g->vkDestroyShaderModule(g->dev, sm, NULL);
    janas_g_free_stage(g, &st);
    return wrong;
}

void janas_g_fma_probe(struct janas_gpu *g)
{
    const char *e = getenv("JANAS_GPU_XFMA");
    int forced = e && *e ? atoi(e) > 0 : -1;
    g->xfma = 0;
    int plain = forced < 0 ? probe(g) : -1;
    int verbose = getenv("JANAS_GPU_VERBOSE") != NULL;
    if (forced == 0 || (forced < 0 && plain <= 0)) {
        if (verbose)
            fprintf(stderr, "gpu fma: %s\n",
                    forced == 0  ? "the shaders' own (JANAS_GPU_XFMA=0)"
                    : plain == 0 ? "rounded once"
                                 : "probe failed");
        return;
    }
    int with = -1;
    if (g->has_f64) {
        g->xfma = 1;
        with = probe(g);
        if (with != 0 && forced < 0)
            g->xfma = 0;
    }
    size_t n = strlen(g->about);
    snprintf(g->about + n, sizeof(g->about) - n, "; fma %s",
             forced > 0   ? (g->xfma ? "in float64 (JANAS_GPU_XFMA=1)"
                                     : "the shaders' own (no float64)")
             : g->xfma    ? "split by the driver, rounded once in float64"
             : g->has_f64 ? "split by the driver (float64 did not mend it)"
                          : "split by the driver (no float64)");
    if (verbose || (forced < 0 && !g->xfma))
        fprintf(stderr,
                "gpu fma: %d of the probe's triples split by the driver; %s "
                "(%d wrong with them)\n",
                plain,
                g->xfma ? "the shaders' float64 variants" : "the shaders' own",
                with);
}
#endif