Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

9.4 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
/*
 * model_mamba.c - the Mamba-2 layers of Nemotron-H (model_nh.c), as
 * llama.cpp's build_mamba2_layer computes them.
 *
 * ssm_in brings the normalized input to z (d_inner), xBC (the conv's
 * channels: x, d_inner, then B and C, n_group x d_state each) and dt (a
 * number a head), each times the matrix's own scale. A causal conv of
 * d_conv taps with a bias, then SiLU, runs over xBC. Head h (of n_vh, each
 * ssm_hd numbers of x, in group g = h / (n_vh / n_group)) keeps a state of
 * ssm_hd rows of d_state:
 *
 *   dt' = softplus(dt + dt_bias),  dA = exp(dt' A)
 *   s[r][i] = s[r][i] dA + B[i] x[r] dt',  y[r] = sum_i s[r][i] C[i]
 *
 * then y + D x, times SiLU(z), an RMS norm (with weights) over each group
 * of d_inner / n_group numbers, and ssm_out, times its scale.
 *
 * The state is in float, in ssm_buf (two uint16 units a number), with the
 * two buffers, the replay of logged tokens and the log of DeltaNet's
 * layers (model_rec.c): log_mixed holds xBC before the conv, log_beta dt'
 * and log_decay dA.
 */
#define _GNU_SOURCE
#include <math.h>
#include <stdlib.h>
#include <string.h>

#include "matvec.h"
#include "vmath.h"

#include "model_impl.h"

struct mb_job {
    struct janas_llm_model *m;
    const struct layer *ly;
    uint32_t n;
};

/* ggml's softplus: x past 20 as it is */
static inline float softplus(float x)
{
    return x > 20.0f ? x : log1pf(expf(x));
}

/* 1. The conv over xBC, item = 8 channels across all the rows (the
      replayed ones, then the call's), as DeltaNet's: the base buffer moves
      past the replayed rows, the call's state goes to the other. */
static void conv_worker(void *arg, int tid, int n_threads)
{
    struct mb_job *a = arg;
    struct janas_llm_model *m = a->m;
    uint32_t cc = m->conv_ch, dc = m->d_conv, rp = m->replay;
    size_t sz = (size_t)m->n_rec * (dc - 1) * cc;
    float *src = m->conv_buf +
                 (size_t)(m->rec_cont ? 1 - m->src : m->src) * sz +
                 (size_t)a->ly->slot * (dc - 1) * cc;
    float *dst = m->conv_buf + (size_t)(1 - m->src) * sz +
                 (size_t)a->ly->slot * (dc - 1) * cc;
    const float *wt = a->ly->conv_t, *bias = a->ly->conv_b;
    for (uint32_t c0 = (uint32_t)tid * 8; c0 < cc;
         c0 += (uint32_t)n_threads * 8)
        for (uint32_t c = c0; c < c0 + 8 && c < cc; c++) {
            float win[MAX_CONV];
            for (uint32_t i = 0; i + 1 < dc; i++)
                win[i] = src[(size_t)i * cc + c];
            for (uint32_t t = 0; t < rp + a->n; t++) {
                if (t == rp && rp > 0)
                    for (uint32_t i = 0; i + 1 < dc; i++)
                        src[(size_t)i * cc + c] = win[i];
                win[dc - 1] = m->mixed[(size_t)t * cc + c];
                float s = 0.0f;
                for (uint32_t i = 0; i < dc; i++)
                    s += win[i] * wt[(size_t)i * cc + c];
                s += bias[c];
                m->conv_out[(size_t)t * cc + c] = s / (1.0f + expf(-s));
                for (uint32_t i = 0; i + 1 < dc; i++)
                    win[i] = win[i + 1];
            }
            for (uint32_t i = 0; i + 1 < dc; i++)
                dst[(size_t)i * cc + c] = win[i];
        }
}

/* 2. The scan, item = a head: its state rows stay in one core's cache for
      all the rows of the call. y goes to att (with D x added). */
static void scan_worker(void *arg, int tid, int n_threads)
{
    struct mb_job *a = arg;
    struct janas_llm_model *m = a->m;
    uint32_t nh = m->n_vh, hd = m->ssm_hd, ds = m->ds, cc = m->conv_ch;
    uint32_t hpg = nh / m->n_kh, di = m->d_inner;
    size_t per = (size_t)nh * hd * ds; /* floats a layer */
    size_t sz = (size_t)m->n_rec * per;
    float *base = (float *)(void *)m->ssm_buf;
    const float *dd = a->ly->ssm_dd;
    for (uint32_t h = (uint32_t)tid; h < nh; h += (uint32_t)n_threads) {
        uint32_t g = h / hpg;
        size_t off = (size_t)a->ly->slot * per + (size_t)h * hd * ds;
        float *M =
            base + (size_t)(m->rec_cont ? 1 - m->src : m->src) * sz + off;
        float *D = base + (size_t)(1 - m->src) * sz + off;
        for (uint32_t t = 0; t < m->replay + a->n; t++) {
            if (t == m->replay) {
                if (M != D)
                    memcpy(D, M, (size_t)hd * ds * sizeof(float));
                M = D;
            }
            const float *co = m->conv_out + (size_t)t * cc;
            const float *x = co + (size_t)h * hd;
            const float *B = co + di + (size_t)g * ds;
            const float *C = co + di + (size_t)m->n_kh * ds + (size_t)g * ds;
            float dt = m->beta[(size_t)t * nh + h];
            float dA = m->decay[(size_t)t * nh + h];
            float *y = t < m->replay ? NULL
                                     : m->att + (size_t)(t - m->replay) * di +
                                           (size_t)h * hd;
            for (uint32_t r = 0; r < hd; r++) {
                float *s = M + (size_t)r * ds;
                float xdt = x[r] * dt, sum = 0.0f;
                for (uint32_t i = 0; i < ds; i++) {
                    float st = s[i] * dA + B[i] * xdt;
                    sum += st * C[i];
                    s[i] = st;
                }
                if (y)
                    y[r] = sum + x[r] * dd[h];
            }
        }
    }
}

/* 3. y times SiLU(z), then the RMS norm of each group, item = (token,
      group). */
static void gate_worker(void *arg, int tid, int n_threads)
{
    struct mb_job *a = arg;
    struct janas_llm_model *m = a->m;
    uint32_t ng = m->n_kh, gw = m->d_inner / ng;
    const float *w = f32(m, a->ly->ssm_norm);
    for (uint32_t it = (uint32_t)tid; it < a->n * ng;
         it += (uint32_t)n_threads) {
        uint32_t j = it / ng, g = it % ng;
        float *o = m->att + (size_t)j * m->d_inner + (size_t)g * gw;
        const float *z = m->big + (size_t)j * m->qkvz_dim + (size_t)g * gw;
        for (uint32_t i = 0; i < gw; i++)
            o[i] *= z[i] / (1.0f + expf(-z[i]));
        rms_norm(o, o, w + (size_t)g * gw, gw, m->eps);
    }
}

void janas_m_mamba_layer(struct janas_llm_model *m, const struct layer *ly,
                         uint32_t n)
{
    uint32_t dm = m->d_model, nh = m->n_vh, cc = m->conv_ch;
    uint32_t di = m->d_inner, qz = m->qkvz_dim;
    size_t nbi = di / JANAS_QK;
    double ta = now();
    struct janas_matvec_task t = {.type = (int)ly->ssm_in->type,
                                  .w = data(m, ly->ssm_in),
                                  .x = m->xq,
                                  .y = m->big,
                                  .rows = qz,
                                  .cols = dm,
                                  .n_vec = n};
    janas_m_products(m, janas_m_gpu(m, 0), &t, 1);
    if (ly->in_scale != 1.0f)
        for (size_t i = 0; i < (size_t)n * qz; i++)
            m->big[i] *= ly->in_scale;
    uint32_t rp = m->replay;
    size_t L = JANAS_LLM_MAX_BLOCK;
    float *lm = m->log_mixed + (size_t)ly->slot * L * cc;
    float *lb = m->log_beta + (size_t)ly->slot * L * nh;
    float *ld = m->log_decay + (size_t)ly->slot * L * nh;
    /* rows 0 .. rp - 1: the logged tokens to replay, then this call's */
    memcpy(m->mixed, lm, (size_t)rp * cc * sizeof(float));
    memcpy(m->beta, lb, (size_t)rp * nh * sizeof(float));
    memcpy(m->decay, ld, (size_t)rp * nh * sizeof(float));
    const float *dtb = f32(m, ly->dt_bias), *A = f32(m, ly->ssm_a);
    for (uint32_t j = 0; j < n; j++) {
        const float *src = m->big + (size_t)j * qz;
        memcpy(m->mixed + (size_t)(rp + j) * cc, src + di, cc * sizeof(float));
        for (uint32_t h = 0; h < nh; h++) {
            float dt = softplus(src[di + cc + h] + dtb[h]);
            m->beta[(size_t)(rp + j) * nh + h] = dt;
            m->decay[(size_t)(rp + j) * nh + h] = expf(dt * A[h]);
        }
    }
    double tb = now();
    m->phase[JANAS_PH_QKV] += tb - ta;
    trace_row(m, "xBC", m->mixed + (size_t)(rp + n - 1) * cc, cc);
    struct mb_job mj = {m, ly, n};
    janas_pool_run(m->compute, conv_worker, &mj);
    trace_row(m, "conv_out", m->conv_out + (size_t)(rp + n - 1) * cc, cc);
    janas_pool_run(m->compute, scan_worker, &mj);
    trace_row(m, "ssm_y", m->att + (size_t)(n - 1) * di, di);
    janas_pool_run(m->compute, gate_worker, &mj);
    trace_row(m, "ssm_gated", m->att + (size_t)(n - 1) * di, di);
    /* logged for a call that may be taken back; a longer one is not */
    if (n <= JANAS_LLM_MAX_BLOCK) {
        memcpy(lm, m->mixed + (size_t)rp * cc, (size_t)n * cc * sizeof(float));
        memcpy(lb, m->beta + (size_t)rp * nh, (size_t)n * nh * sizeof(float));
        memcpy(ld, m->decay + (size_t)rp * nh, (size_t)n * nh * sizeof(float));
    }
    double tc = now();
    m->phase[JANAS_PH_SSM] += tc - tb;
    for (uint32_t j = 0; j < n; j++)
        janas_q8k_quantize(m->att + (size_t)j * di, m->attq + j * nbi, di);
    struct janas_matvec_task to = {.type = (int)ly->ssm_out->type,
                                   .w = data(m, ly->ssm_out),
                                   .x = m->attq,
                                   .y = m->xn,
                                   .rows = dm,
                                   .cols = di,
                                   .n_vec = n};
    janas_m_products(m, janas_m_gpu(m, 0), &to, 1);
    float os = ly->out_scale2;
    for (size_t i = 0; i < (size_t)n * dm; i++)
        m->x[i] += m->xn[i] * os;
    trace_row(m, "residual", m->x + (size_t)(n - 1) * dm, dm);
    m->phase[JANAS_PH_WO] += now() - tc;
}