Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

16.1 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
/*
 * model_ple.c - Gemma 4's per-layer embeddings (E2B, E4B).
 *
 * Every token brings, besides its embedding, a few numbers for each layer
 * (ple_dim, 256): a row of a table of their own (n_layer x ple_dim numbers
 * per token, as large as half of E2B's file) plus a projection of the
 * embedding, normalized layer by layer. After its feed-forward, each layer
 * opens a branch of the residual: gelu(gate x) times its own numbers,
 * projected back to the model's width, normalized, added.
 *
 * The table is read a row at a time from the file, when a token needs it:
 * a few kilobytes a token - or held in memory where the machine has room
 * for it (janas_llm_options ple_ram).
 */
#define _GNU_SOURCE
#include <math.h>
#include <stdatomic.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#include "bf16.h"
#include "matvec.h"
#include "vmath.h"

#include "model_impl.h"

/* f32 rounded to bf16 as ggml does it (to nearest, ties to even), kept as
   f32: the embedding meets the bf16 projection at bf16, as in llama.cpp */
static float round_bf16(float f)
{
    uint32_t u = janas_f2u(f);
    if ((u & 0x7fffffffu) > 0x7f800000u) /* NaN */
        return f;
    u = (u + (0x7fffu + ((u >> 16) & 1u))) & 0xffff0000u;
    return janas_u2f(u);
}

/* rows claimed a few at a time from a shared counter, as the quantized
   products do: equal shares left the performance cores waiting for the
   efficiency cores */
#define PLE_CHUNK 16

/* the bf16 and Q8_0 products' rows: a share of those left, as matvec.c
   claims them - 4 to 256, a multiple of 4 for the four rows a single
   token's kernels take at once - so that a matrix of a few hundred rows
   (a mix's down, 320) does not end on threads left waiting for the last
   fixed chunks; r0, or rows when there is nothing left, and the end in
   *end */
static uint32_t claim_rows(atomic_uint *next, uint32_t rows, int n_threads,
                           uint32_t *end)
{
    uint32_t seen = atomic_load_explicit(next, memory_order_relaxed);
    if (seen >= rows)
        return rows;
    uint32_t c = (rows - seen) / (2u * (uint32_t)n_threads);
    c = c < 4 ? 4 : c > 256 ? 256 : c & ~3u;
    uint32_t r0 = atomic_fetch_add_explicit(next, c, memory_order_relaxed);
    if (r0 >= rows)
        return rows;
    *end = r0 + c < rows ? r0 + c : rows;
    return r0;
}

struct bf16_job {
    atomic_uint next;
    const uint16_t *w;
    const float *x; /* n rows of cols, rounded to bf16 */
    float *y;       /* n rows of rows */
    uint32_t rows, cols, n;
    float scale;
};

#if defined(__x86_64__)
#include <immintrin.h>
#endif

static int have_avx2(void)
{
#if defined(__x86_64__)
    return __builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma");
#else
    return 0;
#endif
}

/* rows claimed by the threads, each row read once for all the tokens:
   four tokens at a time against a row, those left (a reply's one) against
   four rows at a time, so that the sums' chains overlap */
static void bf16_worker(void *arg, int tid, int n_threads)
{
    (void)tid;
    struct bf16_job *b = arg;
    int v = janas_bf16_lanes();
    uint32_t n4 = v ? b->n / 4 * 4 : 0;
    for (;;) {
        uint32_t r1, r0 = claim_rows(&b->next, b->rows, n_threads, &r1);
        if (r0 >= b->rows)
            break;
        for (uint32_t r = r0; r < r1; r++) {
            const uint16_t *w = b->w + (size_t)r * b->cols;
            for (uint32_t j = 0; j < n4; j += 4) {
                const float *x[4];
                float o[4];
                for (int t = 0; t < 4; t++)
                    x[t] = b->x + (size_t)(j + t) * b->cols;
                janas_bf16_dot4(w, x, b->cols, o);
                for (int t = 0; t < 4; t++)
                    b->y[(size_t)(j + t) * b->rows + r] = o[t] * b->scale;
            }
        }
        for (uint32_t j = n4; j < b->n; j++) {
            const float *x = b->x + (size_t)j * b->cols;
            float *y = b->y + (size_t)j * b->rows;
            uint32_t r = r0;
            for (; v && r + 4 <= r1; r += 4) {
                float o[4];
                janas_bf16_dot4r(b->w + (size_t)r * b->cols, b->cols, x,
                                 b->cols, o);
                for (int k = 0; k < 4; k++)
                    y[r + k] = o[k] * b->scale;
            }
            for (; r < r1; r++)
                y[r] = janas_bf16_dot(b->w + (size_t)r * b->cols, x, b->cols) *
                       b->scale;
        }
    }
}

void janas_m_bf16_product(struct janas_llm_model *m, const uint16_t *w,
                          const float *x, float *y, uint32_t rows,
                          uint32_t cols, uint32_t n, float scale)
{
    struct bf16_job b = {.w = w,
                         .x = x,
                         .y = y,
                         .rows = rows,
                         .cols = cols,
                         .n = n,
                         .scale = scale};
    atomic_init(&b.next, 0);
    /* a prompt's block on the NPU, where the pass has it, or on the GPU
       where it can (the fast prompt): summed in another order; else, on a
       GPU that takes single tokens (a card with memory of its own, or
       replies asked of it: janas_gpu_min_n 1), the GPU where the pass has
       it, bit for bit as the CPU's eight lanes - not on an integrated one
       by default, where it is not measured */
    struct janas_gpu *g = janas_m_gpu(m, 0);
    if ((size_t)rows * cols * n < 65536)
        bf16_worker(&b, 0, 1);
    else if (janas_m_npu_dense(m, w, x, y, rows, cols, n, scale) != 0 &&
             janas_gpu_dense_mm(g, 1, w, x, y, rows, cols, n, scale) != 0 &&
             (!g || !janas_bf16_lanes() || janas_gpu_min_n(g) > 1 ||
              janas_gpu_bf16_mv(g, w, x, y, rows, cols, n, scale) != 0))
        janas_pool_run(m->compute, bf16_worker, &b);
}

/* the same for Q8_0 weights (janas_q8w_dot): rows of cols / 32 blocks;
   tokens four at a time against a row, those left (a reply's one) against
   four rows at a time, so that the sums' chains overlap */
static void q8w_worker(void *arg, int tid, int n_threads)
{
    (void)tid;
    struct bf16_job *b = arg;
    size_t rb = (size_t)b->cols / 32 * 34;
    uint32_t n4 = b->n / 4 * 4;
    for (;;) {
        uint32_t r1, r0 = claim_rows(&b->next, b->rows, n_threads, &r1);
        if (r0 >= b->rows)
            break;
        for (uint32_t r = r0; r < r1; r++) {
            const uint8_t *w = (const uint8_t *)b->w + r * rb;
            for (uint32_t j = 0; j < n4; j += 4) {
                const float *x[4];
                float o[4];
                for (int t = 0; t < 4; t++)
                    x[t] = b->x + (size_t)(j + t) * b->cols;
                janas_q8w_dot4(w, x, b->cols, o);
                for (int t = 0; t < 4; t++)
                    b->y[(size_t)(j + t) * b->rows + r] = o[t] * b->scale;
            }
        }
        for (uint32_t j = n4; j < b->n; j++) {
            const float *x = b->x + (size_t)j * b->cols;
            float *y = b->y + (size_t)j * b->rows;
            uint32_t r = r0;
            for (; r + 4 <= r1; r += 4) {
                float o[4];
                janas_q8w_dot4r((const uint8_t *)b->w + r * rb, rb, x, b->cols,
                                o);
                for (int k = 0; k < 4; k++)
                    y[r + k] = o[k] * b->scale;
            }
            for (; r < r1; r++)
                y[r] =
                    janas_q8w_dot((const uint8_t *)b->w + r * rb, x, b->cols) *
                    b->scale;
        }
    }
}

void janas_m_q8w_product(struct janas_llm_model *m, const void *w,
                         const float *x, float *y, uint32_t rows, uint32_t cols,
                         uint32_t n, float scale)
{
    struct bf16_job b = {.w = w,
                         .x = x,
                         .y = y,
                         .rows = rows,
                         .cols = cols,
                         .n = n,
                         .scale = scale};
    atomic_init(&b.next, 0);
    if ((size_t)rows * cols * n < 65536)
        q8w_worker(&b, 0, 1);
    else
        janas_pool_run(m->compute, q8w_worker, &b);
}

float janas_m_round_bf16(float f)
{
    return round_bf16(f);
}

struct f32_job {
    atomic_uint next;
    const float *w, *x;
    float *y;
    uint32_t rows, cols, n;
};

#if defined(__x86_64__)
__attribute__((target("avx2,fma"))) static float
dot_f32_avx2(const float *w, const float *x, uint32_t n)
{
    __m256 a0 = _mm256_setzero_ps(), a1 = _mm256_setzero_ps();
    uint32_t i = 0;
    for (; i + 16 <= n; i += 16) {
        a0 =
            _mm256_fmadd_ps(_mm256_loadu_ps(w + i), _mm256_loadu_ps(x + i), a0);
        a1 = _mm256_fmadd_ps(_mm256_loadu_ps(w + i + 8),
                             _mm256_loadu_ps(x + i + 8), a1);
    }
    float r = janas_hsum8(_mm256_add_ps(a0, a1));
    for (; i < n; i++)
        r += w[i] * x[i];
    return r;
}

/* four tokens against one row, as dot_f32_avx2 for each: the same bits */
__attribute__((target("avx2,fma"))) static void
dot4_f32_avx2(const float *w, const float *const x[4], uint32_t n, float out[4])
{
    __m256 a0[4], a1[4];
    for (int t = 0; t < 4; t++)
        a0[t] = a1[t] = _mm256_setzero_ps();
    uint32_t i = 0;
    for (; i + 16 <= n; i += 16) {
        __m256 w0 = _mm256_loadu_ps(w + i), w1 = _mm256_loadu_ps(w + i + 8);
        for (int t = 0; t < 4; t++) {
            a0[t] = _mm256_fmadd_ps(w0, _mm256_loadu_ps(x[t] + i), a0[t]);
            a1[t] = _mm256_fmadd_ps(w1, _mm256_loadu_ps(x[t] + i + 8), a1[t]);
        }
    }
    for (int t = 0; t < 4; t++) {
        float r = janas_hsum8(_mm256_add_ps(a0[t], a1[t]));
        for (uint32_t k = i; k < n; k++)
            r += w[k] * x[t][k];
        out[t] = r;
    }
}
#endif

/* one dot product, the same for a row and a token whatever the pass */
static float dot_f32(const float *w, const float *x, uint32_t n)
{
#if defined(__x86_64__)
    if (__builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma"))
        return dot_f32_avx2(w, x, n);
#endif
    float r = 0.0f;
    for (uint32_t i = 0; i < n; i++)
        r += w[i] * x[i];
    return r;
}

/* rows in equal runs, each read once for all the tokens: the f32
   matrices of E2B's and E4B's branches are 2.6 MB a layer */
static void f32_worker(void *arg, int tid, int n_threads)
{
    (void)tid;
    (void)n_threads;
    struct f32_job *b = arg;
    int v = have_avx2();
    for (;;) {
        uint32_t r0 = atomic_fetch_add(&b->next, PLE_CHUNK);
        if (r0 >= b->rows)
            break;
        uint32_t r1 = r0 + PLE_CHUNK < b->rows ? r0 + PLE_CHUNK : b->rows;
        for (uint32_t r = r0; r < r1; r++) {
            const float *w = b->w + (size_t)r * b->cols;
            uint32_t j = 0;
#if defined(__x86_64__)
            for (; v && j + 4 <= b->n; j += 4) {
                const float *x[4];
                float o[4];
                for (int t = 0; t < 4; t++)
                    x[t] = b->x + (size_t)(j + t) * b->cols;
                dot4_f32_avx2(w, x, b->cols, o);
                for (int t = 0; t < 4; t++)
                    b->y[(size_t)(j + t) * b->rows + r] = o[t];
            }
#endif
            for (; j < b->n; j++)
                b->y[(size_t)j * b->rows + r] =
                    dot_f32(w, b->x + (size_t)j * b->cols, b->cols);
        }
    }
}

/* y (n rows of t's rows) = t x (n rows of cols): f32 weights as they are,
   quantized ones on x put in Q8_K (q, n * cols / 256 blocks) */
static void product(struct janas_llm_model *m, const struct janas_jns_tensor *t,
                    const float *x, struct janas_block_q8k *q, float *y,
                    uint32_t n)
{
    uint32_t cols = (uint32_t)t->dims[0], rows = (uint32_t)t->dims[1];
    if (t->type == 0) {
        if (janas_gpu_dense_mm(janas_m_gpu(m, 0), 0, f32(m, t), x, y, rows,
                               cols, n, 1.0f) == 0)
            return;
        struct f32_job b = {
            .w = f32(m, t), .x = x, .y = y, .rows = rows, .cols = cols, .n = n};
        atomic_init(&b.next, 0);
        janas_pool_run(m->compute, f32_worker, &b);
        return;
    }
    size_t nb = cols / JANAS_QK;
    for (uint32_t j = 0; j < n; j++)
        janas_q8k_quantize(x + (size_t)j * cols, q + j * nb, cols);
    struct janas_matvec_task tk = {.type = (int)t->type,
                                   .w = data(m, t),
                                   .x = q,
                                   .y = y,
                                   .rows = rows,
                                   .cols = cols,
                                   .n_vec = n};
    janas_m_products(m, janas_m_gpu(m, 0), &tk, 1);
}

/*
 * The block's per-layer inputs, from the embeddings in x (already scaled by
 * sqrt(d_model)): the projection over sqrt(d_model), each layer's part
 * normalized, plus the table's row times sqrt(ple_dim), all over sqrt(2).
 */
int janas_m_ple_inputs(struct janas_llm_model *m, const int32_t *tokens,
                       uint32_t n, uint32_t pos0)
{
    uint32_t dm = m->d_model, P = m->ple_dim, W = m->n_layer * P;
    float *xb = m->xn; /* the embeddings at bf16 */
    for (size_t i = 0; i < (size_t)n * dm; i++)
        xb[i] = round_bf16(m->x[i]);
    struct bf16_job b = {.w = data(m, m->ple_model),
                         .x = xb,
                         .y = m->ple,
                         .rows = W,
                         .cols = dm,
                         .n = n,
                         .scale = 1.0f / sqrtf((float)dm)};
    atomic_init(&b.next, 0);
    if (n < 2)
        bf16_worker(&b, 0, 1);
    else if (janas_gpu_dense_mm(janas_m_gpu(m, 0), 1, b.w, b.x, b.y, b.rows,
                                b.cols, n, b.scale) != 0)
        janas_pool_run(m->compute, bf16_worker, &b);
    const float *nw = f32(m, m->ple_norm);
    size_t row_bytes =
        W / JANAS_QK * janas_qtype_block_size((int)m->ple_tok->type);
    uint8_t *raw = (uint8_t *)(m->ple_row + W); /* the row as read */
    float es = sqrtf((float)P), is2 = 1.0f / sqrtf(2.0f);
    for (uint32_t j = 0; j < n; j++) {
        /* a picture's token takes the padding's row (id 0), as both the
           reference and llama.cpp do */
        int32_t tok =
            m->n_media && janas_m_media_row(m, pos0 + j) ? 0 : tokens[j];
        off_t at = (off_t)(m->ple_tok->offset + (uint64_t)tok * row_bytes);
        const uint8_t *src = raw;
        if (m->ple_ram) /* the table in memory (janas_llm_options) */
            src = (const uint8_t *)data(m, m->ple_tok) +
                  (uint64_t)tok * row_bytes;
        for (size_t done = 0; !m->ple_ram && done < row_bytes;) {
            ssize_t r =
                pread(m->j.fd, raw + done, row_bytes - done, at + (off_t)done);
            if (r <= 0)
                return -1;
            done += (size_t)r;
        }
        janas_dequantize((int)m->ple_tok->type, src, m->ple_row, W);
        float *p = m->ple + (size_t)j * W;
        for (uint32_t l = 0; l < m->n_layer; l++) {
            float *pl = p + (size_t)l * P;
            const float *el = m->ple_row + (size_t)l * P;
            rms_norm(pl, pl, nw, P, m->eps);
            for (uint32_t i = 0; i < P; i++)
                pl[i] = (pl[i] + el[i] * es) * is2;
        }
    }
    trace_row(m, "inp_per_layer", m->ple + (size_t)(n - 1) * W, P);
    return 0;
}

/*
 * Layer l's branch, on the residual in x after the feed-forward, then the
 * layer's output scale: x = (x + norm(proj(gelu(gate x) * input))) * s.
 */
void janas_m_ple_layer(struct janas_llm_model *m, const struct layer *ly,
                       uint32_t l, uint32_t n)
{
    uint32_t dm = m->d_model, P = m->ple_dim, W = m->n_layer * P;
    product(m, ly->ple_gate, m->x, m->xq, m->ple_g, n);
    for (uint32_t j = 0; j < n; j++) {
        float *g = m->ple_g + (size_t)j * P;
        const float *in = m->ple + (size_t)j * W + (size_t)l * P;
        for (uint32_t i = 0; i < P; i++)
            g[i] = janas_gelu_mul_det(g[i], in[i]);
    }
    product(m, ly->ple_proj, m->ple_g, m->ple_q, m->xn, n);
    const float *pw = f32(m, ly->ple_post);
    for (uint32_t j = 0; j < n; j++) {
        float *x = m->x + (size_t)j * dm, *y = m->xn + (size_t)j * dm;
        rms_norm(y, y, pw, dm, m->eps);
        for (uint32_t i = 0; i < dm; i++)
            x[i] = (x[i] + y[i]) * ly->out_scale;
    }
    trace_row(m, "per_layer_out", m->x + (size_t)(n - 1) * dm, dm);
}