Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

12.9 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
/*
 * model_attn.c - the attention layers of the forward pass: norms, q/k/v,
 * RoPE, the KV cache, the output projection (see model.h).
 */
#define _GNU_SOURCE
#include "model.h"

#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>

#include "attention.h"
#include "common/pool.h"
#include "common/sysinfo.h"
#include "deltanet.h"
#include "tuner.h"
#include "gpu.h"
#include "matvec.h"
#include "quant.h"
#include "vmath.h"

#include "model_impl.h"

/* NeoX rotation of the first n dimensions of a head, pairs (i, i + n/2),
   with per-token tables; the other dimensions are left as they are. */
static void rope_neox(float *v, uint32_t n, const float *cs, const float *sn)
{
    uint32_t half = n / 2;
    for (uint32_t i = 0; i < half; i++) {
        float a = v[i], b = v[i + half];
        v[i] = a * cs[i] - b * sn[i];
        v[i + half] = a * sn[i] + b * cs[i];
    }
}

/* rms norm of x into xn, then xn to Q8_K in xq, tokens first .. n - 1 */
static void norm_quant_tok(struct janas_llm_model *m, const float *w,
                           uint32_t j)
{
    uint32_t dm = m->d_model;
    rms_norm(m->xn + (size_t)j * dm, m->x + (size_t)j * dm, w, dm, m->eps);
    janas_q8k_quantize(m->xn + (size_t)j * dm, m->xq + j * (dm / JANAS_QK), dm);
}

static void norm_quant_worker(void *arg, int tid, int n_threads)
{
    struct tok_job *t = arg;
    for (uint32_t j = t->first + (uint32_t)tid; j < t->n;
         j += (uint32_t)n_threads)
        norm_quant_tok(t->m, t->w, j);
}

void janas_m_norm_quant(struct janas_llm_model *m, const float *w,
                        uint32_t first, uint32_t n)
{
    if (n - first < 2) {
        for (uint32_t j = first; j < n; j++)
            norm_quant_tok(m, w, j);
        return;
    }
    struct tok_job t = {.m = m, .w = w, .first = first, .n = n};
    janas_pool_run(m->compute, norm_quant_worker, &t);
}

/*
 * After the q, k, v products, item (token j, head h): the heads of q are
 * normalized and turned (qwen3next: split from their output gate first),
 * those of k normalized, turned and put in the cache at eight bits with v.
 */
static void qkv_item(const struct tok_job *t, uint32_t j, uint32_t h)
{
    struct janas_llm_model *m = t->m;
    const struct layer *ly = t->ly;
    uint32_t hd = ly->hd, half = ly->n_rot / 2;
    uint32_t qd = m->n_head * hd, kvd = ly->n_kv * hd;
    const float *cs =
        (ly->swa ? m->rope_cos_swa : m->rope_cos) + (size_t)j * half;
    const float *sn =
        (ly->swa ? m->rope_sin_swa : m->rope_sin) + (size_t)j * half;
    if (h < m->n_head) {
        float *v = m->q + (size_t)j * qd + (size_t)h * hd;
        if (m->a->attn_gate) {
            const float *src = m->big + (size_t)j * 2 * qd + 2 * h * hd;
            memcpy(v, src, hd * sizeof(float));
            memcpy(m->qgate + (size_t)j * qd + (size_t)h * hd, src + hd,
                   hd * sizeof(float));
        }
        if (ly->q_norm)
            rms_norm(v, v, f32(m, ly->q_norm), hd, m->eps);
        rope_neox(v, ly->n_rot, cs, sn);
        return;
    }
    h -= m->n_head;
    float *kv = m->kk + (size_t)j * kvd + (size_t)h * hd;
    if (ly->k_norm)
        rms_norm(kv, kv, f32(m, ly->k_norm), hd, m->eps);
    rope_neox(kv, ly->n_rot, cs, sn);
    uint32_t p = t->pos0 + j, row = t->ring ? p % t->cap : p;
    size_t at = ((size_t)h * t->cap + row) * hd;
    size_t as = (size_t)h * t->cap + row;
    float *vr = m->vv + (size_t)j * kvd + (size_t)h * hd;
    if (m->a->swa) { /* Gemma 4: V normalized, without weights */
        double ss = 0.0;
        for (uint32_t i = 0; i < hd; i++)
            ss += (double)vr[i] * vr[i];
        float sc = 1.0f / sqrtf((float)(ss / hd) + m->eps);
        for (uint32_t i = 0; i < hd; i++)
            vr[i] *= sc;
    }
    /* eight bits with a scale of their own: the largest of the head's
       numbers becomes 127, so the error is under half a step of a range
       the numbers themselves set */
    float mk = 0.0f, mv = 0.0f;
    for (uint32_t i = 0; i < hd; i++) {
        float a = fabsf(kv[i]), b = fabsf(vr[i]);
        if (a > mk)
            mk = a;
        if (b > mv)
            mv = b;
    }
    if (m->kv16) { /* sixteen bits: the largest becomes 32767 */
        t->kc_scale[as] = mk / 32767.0f;
        t->vc_scale[as] = mv / 32767.0f;
        float ik = mk > 0 ? 32767.0f / mk : 0.0f;
        float iv = mv > 0 ? 32767.0f / mv : 0.0f;
        int16_t *kc = (int16_t *)(void *)t->kc, *vc = (int16_t *)(void *)t->vc;
        for (uint32_t i = 0; i < hd; i++) {
            kc[at + i] = (int16_t)lrintf(kv[i] * ik);
            vc[at + i] = (int16_t)lrintf(vr[i] * iv);
        }
        return;
    }
    t->kc_scale[as] = mk / 127.0f;
    t->vc_scale[as] = mv / 127.0f;
    float ik = mk > 0 ? 127.0f / mk : 0.0f;
    float iv = mv > 0 ? 127.0f / mv : 0.0f;
    for (uint32_t i = 0; i < hd; i++) {
        t->kc[at + i] = (int8_t)lrintf(kv[i] * ik);
        t->vc[at + i] = (int8_t)lrintf(vr[i] * iv);
    }
}

static void qkv_worker(void *arg, int tid, int n_threads)
{
    struct tok_job *t = arg;
    /* a layer without keys and values of its own: its queries alone */
    uint32_t nh = t->m->n_head + (t->ly->wk ? t->ly->n_kv : 0);
    for (uint32_t i = (uint32_t)tid; i < t->n * nh; i += (uint32_t)n_threads)
        qkv_item(t, i / nh, i % nh);
}

/* the attention output of token j gated (qwen3next) and put in Q8_K */
static void att_out_tok(struct janas_llm_model *m, const struct layer *ly,
                        uint32_t j)
{
    uint32_t qd = m->n_head * ly->hd;
    float *a = m->att + (size_t)j * qd;
    if (m->a->attn_gate) {
        const float *g = m->qgate + (size_t)j * qd;
        for (uint32_t i = 0; i < qd; i++)
            a[i] *= sigmoid(g[i]);
    }
    janas_q8k_quantize(a, m->attq + j * (qd / JANAS_QK), qd);
}

static void att_out_worker(void *arg, int tid, int n_threads)
{
    struct tok_job *t = arg;
    for (uint32_t j = (uint32_t)tid; j < t->n; j += (uint32_t)n_threads)
        att_out_tok(t->m, t->ly, j);
}

/* xn = rms_norm(xn) * w in place, token by token (sandwich norms) */
static void post_norm_worker(void *arg, int tid, int n_threads)
{
    struct tok_job *t = arg;
    uint32_t dm = t->m->d_model;
    for (uint32_t j = (uint32_t)tid; j < t->n; j += (uint32_t)n_threads) {
        float *v = t->m->xn + (size_t)j * dm;
        rms_norm(v, v, t->w, dm, t->m->eps);
    }
}

void janas_m_post_norm(struct janas_llm_model *m, const float *w, uint32_t n)
{
    struct tok_job t = {.m = m, .w = w, .n = n};
    if (n < 2)
        post_norm_worker(&t, 0, 1);
    else
        janas_pool_run(m->compute, post_norm_worker, &t);
}

/* x += xn, token by token */
static void residual_worker(void *arg, int tid, int n_threads)
{
    struct tok_job *t = arg;
    uint32_t dm = t->m->d_model;
    for (uint32_t j = (uint32_t)tid; j < t->n; j += (uint32_t)n_threads) {
        float *x = t->m->x + (size_t)j * dm;
        const float *y = t->m->xn + (size_t)j * dm;
        for (uint32_t i = 0; i < dm; i++)
            x[i] += y[i];
    }
}

void janas_m_residual(struct janas_llm_model *m, uint32_t n)
{
    if (n < 2) {
        for (uint32_t i = 0; i < n * m->d_model; i++)
            m->x[i] += m->xn[i];
        return;
    }
    struct tok_job t = {.m = m, .n = n};
    janas_pool_run(m->compute, residual_worker, &t);
}

/* Attention layer on the normalized block in xn / xq; output added to x. */
void janas_m_attn_layer(struct janas_llm_model *m, const struct layer *ly,
                        uint32_t n, uint32_t pos0)
{
    uint32_t dm = m->d_model, hd = ly->hd;
    uint32_t qd = m->n_head * hd, kvd = ly->n_kv * hd;
    double ta = now();
    /* qwen3next: each query head is followed by its output gate */
    float *qout = m->a->attn_gate ? m->big : m->q;
    /* no k: the keys and values of an earlier layer, already cached */
    int nt = !ly->wk ? 1 : ly->wv ? 3 : 2;
    struct janas_matvec_task t[3] = {{.type = (int)ly->wq->type,
                                      .w = data(m, ly->wq),
                                      .x = m->xq,
                                      .y = qout,
                                      .rows = m->a->attn_gate ? 2 * qd : qd,
                                      .cols = dm,
                                      .n_vec = n},
                                     {.type = ly->wk ? (int)ly->wk->type : 0,
                                      .w = ly->wk ? data(m, ly->wk) : NULL,
                                      .x = m->xq,
                                      .y = m->kk,
                                      .rows = kvd,
                                      .cols = dm,
                                      .n_vec = n},
                                     {.type = ly->wv ? (int)ly->wv->type : 0,
                                      .w = ly->wv ? data(m, ly->wv) : NULL,
                                      .x = m->xq,
                                      .y = m->vv,
                                      .rows = kvd,
                                      .cols = dm,
                                      .n_vec = n}};
    janas_m_products(m, janas_m_gpu(m, 0), t, (size_t)nt);
    if (nt == 2) /* Gemma 4's full layers: V is K, before K's norm */
        memcpy(m->vv, m->kk, (size_t)n * kvd * sizeof(float));
    janas_m_attn_core(m, ly, n, pos0, ta);
    /* qwen4exp: past its budget, the sparse attention - not in its MTP
       block, whose indexer is left unused: dense, as llama.cpp's */
    if (m->ix_heads && ly->ix_q)
        janas_m_qsa(m, ly, n, pos0);
    if (m->npu) /* the NPU's calibration (JANAS_NPU_A8=1) */
        janas_m_npu_calib(m, 0, m->ana_layer, m->att, n, qd, qd);
    double tc = now();
    /* the output gate (qwen3next) goes with the quantization now */
    struct tok_job tj = {.m = m, .ly = ly, .n = n};
    if (n < 2)
        for (uint32_t j = 0; j < n; j++)
            att_out_tok(m, ly, j);
    else
        janas_pool_run(m->compute, att_out_worker, &tj);
    struct janas_matvec_task to = {.type = (int)ly->wo->type,
                                   .w = data(m, ly->wo),
                                   .x = m->attq,
                                   .y = m->xn,
                                   .rows = dm,
                                   .cols = qd,
                                   .n_vec = n};
    janas_m_products(m, janas_m_gpu(m, 0), &to, 1);
    if (ly->post_attn)
        janas_m_post_norm(m, f32(m, ly->post_attn), n);
    janas_m_residual(m, n);
    m->phase[JANAS_PH_WO] += now() - tc;
}

/* The attention of a block whose q, k and v are in m->q, m->kk and m->vv
   (products done): q's and k's norms, RoPE, the keys and values into the
   cache, the attention into m->att. ta: when the layer began (its q, k and
   v counted with it). */
void janas_m_attn_core(struct janas_llm_model *m, const struct layer *ly,
                       uint32_t n, uint32_t pos0, double ta)
{
    uint32_t hd = ly->hd, qd = m->n_head * hd, kvd = ly->n_kv * hd;
    int8_t *kc = m->kcache + m->kv_off[ly->slot];
    int8_t *vc = m->vcache + m->kv_off[ly->slot];
    float *kcs = m->kscale + m->ks_off[ly->slot];
    float *vcs = m->vscale + m->ks_off[ly->slot];
    struct tok_job tj = {.m = m,
                         .ly = ly,
                         .n = n,
                         .pos0 = pos0,
                         .kc_scale = kcs,
                         .vc_scale = vcs,
                         .kc = kc,
                         .vc = vc,
                         .cap = m->kv_cap[ly->slot],
                         .ring = m->kv_ring[ly->slot]};
    janas_pool_run(m->compute, qkv_worker, &tj);
    /* the last token's last head of each, as llama.cpp's graph names them */
    trace_row(m, "Qcur_pos", m->q + (size_t)(n - 1) * qd + (m->n_head - 1) * hd,
              hd);
    trace_row(m, "Kcur_pos",
              m->kk + (size_t)(n - 1) * kvd + (ly->n_kv - 1) * hd, hd);
    trace_row(m, "Vcur", m->vv + (size_t)(n - 1) * kvd + (ly->n_kv - 1) * hd,
              hd);
    double tb = now();
    m->phase[JANAS_PH_QKV] += tb - ta;
    /* attention takes JANAS_LLM_MAX_BLOCK tokens a call; the keys and values
       of the whole block are in the cache already, and each piece sees the
       positions up to its own end, as the block would */
    /* a picture whose tokens see one another: each sees to its end */
    int both = m->n_media && janas_m_media_both(m, pos0, n);
    if (both)
        janas_m_media_last(m, pos0, n, m->att_last);
    for (uint32_t s = 0; s < n; s += JANAS_LLM_MAX_BLOCK) {
        uint32_t len =
            n - s < JANAS_LLM_MAX_BLOCK ? n - s : JANAS_LLM_MAX_BLOCK;
        if (both)
            janas_attn_set_last(ly->swa ? m->attn_swa : m->attn,
                                m->att_last + s);
        janas_attn_run(ly->swa ? m->attn_swa : m->attn, m->compute,
                       m->q + (size_t)s * qd, kc, kcs, vc, vcs, len, pos0 + s,
                       m->att + (size_t)s * qd);
    }
    m->phase[JANAS_PH_ATTN] += now() - tb;
    trace_row(m, "kqv_out", m->att + (size_t)(n - 1) * qd, qd);
}