37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
/*
* model_ple.c - Gemma 4's per-layer embeddings (E2B, E4B).
*
* Every token brings, besides its embedding, a few numbers for each layer
* (ple_dim, 256): a row of a table of their own (n_layer x ple_dim numbers
* per token, as large as half of E2B's file) plus a projection of the
* embedding, normalized layer by layer. After its feed-forward, each layer
* opens a branch of the residual: gelu(gate x) times its own numbers,
* projected back to the model's width, normalized, added.
*
* The table is read a row at a time from the file, when a token needs it:
* a few kilobytes a token - or held in memory where the machine has room
* for it (janas_llm_options ple_ram).
*/
#define _GNU_SOURCE
#include <math.h>
#include <stdatomic.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include "bf16.h"
#include "matvec.h"
#include "vmath.h"
#include "model_impl.h"
/* f32 rounded to bf16 as ggml does it (to nearest, ties to even), kept as
f32: the embedding meets the bf16 projection at bf16, as in llama.cpp */
static float round_bf16(float f)
{
uint32_t u = janas_f2u(f);
if ((u & 0x7fffffffu) > 0x7f800000u) /* NaN */
return f;
u = (u + (0x7fffu + ((u >> 16) & 1u))) & 0xffff0000u;
return janas_u2f(u);
}
/* rows claimed a few at a time from a shared counter, as the quantized
products do: equal shares left the performance cores waiting for the
efficiency cores */
#define PLE_CHUNK 16
/* the bf16 and Q8_0 products' rows: a share of those left, as matvec.c
claims them - 4 to 256, a multiple of 4 for the four rows a single
token's kernels take at once - so that a matrix of a few hundred rows
(a mix's down, 320) does not end on threads left waiting for the last
fixed chunks; r0, or rows when there is nothing left, and the end in
*end */
static uint32_t claim_rows(atomic_uint *next, uint32_t rows, int n_threads,
uint32_t *end)
{
uint32_t seen = atomic_load_explicit(next, memory_order_relaxed);
if (seen >= rows)
return rows;
uint32_t c = (rows - seen) / (2u * (uint32_t)n_threads);
c = c < 4 ? 4 : c > 256 ? 256 : c & ~3u;
uint32_t r0 = atomic_fetch_add_explicit(next, c, memory_order_relaxed);
if (r0 >= rows)
return rows;
*end = r0 + c < rows ? r0 + c : rows;
return r0;
}
struct bf16_job {
atomic_uint next;
const uint16_t *w;
const float *x; /* n rows of cols, rounded to bf16 */
float *y; /* n rows of rows */
uint32_t rows, cols, n;
float scale;
};
#if defined(__x86_64__)
#include <immintrin.h>
#endif
static int have_avx2(void)
{
#if defined(__x86_64__)
return __builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma");
#else
return 0;
#endif
}
/* rows claimed by the threads, each row read once for all the tokens:
four tokens at a time against a row, those left (a reply's one) against
four rows at a time, so that the sums' chains overlap */
static void bf16_worker(void *arg, int tid, int n_threads)
{
(void)tid;
struct bf16_job *b = arg;
int v = janas_bf16_lanes();
uint32_t n4 = v ? b->n / 4 * 4 : 0;
for (;;) {
uint32_t r1, r0 = claim_rows(&b->next, b->rows, n_threads, &r1);
if (r0 >= b->rows)
break;
for (uint32_t r = r0; r < r1; r++) {
const uint16_t *w = b->w + (size_t)r * b->cols;
for (uint32_t j = 0; j < n4; j += 4) {
const float *x[4];
float o[4];
for (int t = 0; t < 4; t++)
x[t] = b->x + (size_t)(j + t) * b->cols;
janas_bf16_dot4(w, x, b->cols, o);
for (int t = 0; t < 4; t++)
b->y[(size_t)(j + t) * b->rows + r] = o[t] * b->scale;
}
}
for (uint32_t j = n4; j < b->n; j++) {
const float *x = b->x + (size_t)j * b->cols;
float *y = b->y + (size_t)j * b->rows;
uint32_t r = r0;
for (; v && r + 4 <= r1; r += 4) {
float o[4];
janas_bf16_dot4r(b->w + (size_t)r * b->cols, b->cols, x,
b->cols, o);
for (int k = 0; k < 4; k++)
y[r + k] = o[k] * b->scale;
}
for (; r < r1; r++)
y[r] = janas_bf16_dot(b->w + (size_t)r * b->cols, x, b->cols) *
b->scale;
}
}
}
void janas_m_bf16_product(struct janas_llm_model *m, const uint16_t *w,
const float *x, float *y, uint32_t rows,
uint32_t cols, uint32_t n, float scale)
{
struct bf16_job b = {.w = w,
.x = x,
.y = y,
.rows = rows,
.cols = cols,
.n = n,
.scale = scale};
atomic_init(&b.next, 0);
/* a prompt's block on the NPU, where the pass has it, or on the GPU
where it can (the fast prompt): summed in another order; else, on a
GPU that takes single tokens (a card with memory of its own, or
replies asked of it: janas_gpu_min_n 1), the GPU where the pass has
it, bit for bit as the CPU's eight lanes - not on an integrated one
by default, where it is not measured */
struct janas_gpu *g = janas_m_gpu(m, 0);
if ((size_t)rows * cols * n < 65536)
bf16_worker(&b, 0, 1);
else if (janas_m_npu_dense(m, w, x, y, rows, cols, n, scale) != 0 &&
janas_gpu_dense_mm(g, 1, w, x, y, rows, cols, n, scale) != 0 &&
(!g || !janas_bf16_lanes() || janas_gpu_min_n(g) > 1 ||
janas_gpu_bf16_mv(g, w, x, y, rows, cols, n, scale) != 0))
janas_pool_run(m->compute, bf16_worker, &b);
}
/* the same for Q8_0 weights (janas_q8w_dot): rows of cols / 32 blocks;
tokens four at a time against a row, those left (a reply's one) against
four rows at a time, so that the sums' chains overlap */
static void q8w_worker(void *arg, int tid, int n_threads)
{
(void)tid;
struct bf16_job *b = arg;
size_t rb = (size_t)b->cols / 32 * 34;
uint32_t n4 = b->n / 4 * 4;
for (;;) {
uint32_t r1, r0 = claim_rows(&b->next, b->rows, n_threads, &r1);
if (r0 >= b->rows)
break;
for (uint32_t r = r0; r < r1; r++) {
const uint8_t *w = (const uint8_t *)b->w + r * rb;
for (uint32_t j = 0; j < n4; j += 4) {
const float *x[4];
float o[4];
for (int t = 0; t < 4; t++)
x[t] = b->x + (size_t)(j + t) * b->cols;
janas_q8w_dot4(w, x, b->cols, o);
for (int t = 0; t < 4; t++)
b->y[(size_t)(j + t) * b->rows + r] = o[t] * b->scale;
}
}
for (uint32_t j = n4; j < b->n; j++) {
const float *x = b->x + (size_t)j * b->cols;
float *y = b->y + (size_t)j * b->rows;
uint32_t r = r0;
for (; r + 4 <= r1; r += 4) {
float o[4];
janas_q8w_dot4r((const uint8_t *)b->w + r * rb, rb, x, b->cols,
o);
for (int k = 0; k < 4; k++)
y[r + k] = o[k] * b->scale;
}
for (; r < r1; r++)
y[r] =
janas_q8w_dot((const uint8_t *)b->w + r * rb, x, b->cols) *
b->scale;
}
}
}
void janas_m_q8w_product(struct janas_llm_model *m, const void *w,
const float *x, float *y, uint32_t rows, uint32_t cols,
uint32_t n, float scale)
{
struct bf16_job b = {.w = w,
.x = x,
.y = y,
.rows = rows,
.cols = cols,
.n = n,
.scale = scale};
atomic_init(&b.next, 0);
if ((size_t)rows * cols * n < 65536)
q8w_worker(&b, 0, 1);
else
janas_pool_run(m->compute, q8w_worker, &b);
}
float janas_m_round_bf16(float f)
{
return round_bf16(f);
}
struct f32_job {
atomic_uint next;
const float *w, *x;
float *y;
uint32_t rows, cols, n;
};
#if defined(__x86_64__)
__attribute__((target("avx2,fma"))) static float
dot_f32_avx2(const float *w, const float *x, uint32_t n)
{
__m256 a0 = _mm256_setzero_ps(), a1 = _mm256_setzero_ps();
uint32_t i = 0;
for (; i + 16 <= n; i += 16) {
a0 =
_mm256_fmadd_ps(_mm256_loadu_ps(w + i), _mm256_loadu_ps(x + i), a0);
a1 = _mm256_fmadd_ps(_mm256_loadu_ps(w + i + 8),
_mm256_loadu_ps(x + i + 8), a1);
}
float r = janas_hsum8(_mm256_add_ps(a0, a1));
for (; i < n; i++)
r += w[i] * x[i];
return r;
}
/* four tokens against one row, as dot_f32_avx2 for each: the same bits */
__attribute__((target("avx2,fma"))) static void
dot4_f32_avx2(const float *w, const float *const x[4], uint32_t n, float out[4])
{
__m256 a0[4], a1[4];
for (int t = 0; t < 4; t++)
a0[t] = a1[t] = _mm256_setzero_ps();
uint32_t i = 0;
for (; i + 16 <= n; i += 16) {
__m256 w0 = _mm256_loadu_ps(w + i), w1 = _mm256_loadu_ps(w + i + 8);
for (int t = 0; t < 4; t++) {
a0[t] = _mm256_fmadd_ps(w0, _mm256_loadu_ps(x[t] + i), a0[t]);
a1[t] = _mm256_fmadd_ps(w1, _mm256_loadu_ps(x[t] + i + 8), a1[t]);
}
}
for (int t = 0; t < 4; t++) {
float r = janas_hsum8(_mm256_add_ps(a0[t], a1[t]));
for (uint32_t k = i; k < n; k++)
r += w[k] * x[t][k];
out[t] = r;
}
}
#endif
/* one dot product, the same for a row and a token whatever the pass */
static float dot_f32(const float *w, const float *x, uint32_t n)
{
#if defined(__x86_64__)
if (__builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma"))
return dot_f32_avx2(w, x, n);
#endif
float r = 0.0f;
for (uint32_t i = 0; i < n; i++)
r += w[i] * x[i];
return r;
}
/* rows in equal runs, each read once for all the tokens: the f32
matrices of E2B's and E4B's branches are 2.6 MB a layer */
static void f32_worker(void *arg, int tid, int n_threads)
{
(void)tid;
(void)n_threads;
struct f32_job *b = arg;
int v = have_avx2();
for (;;) {
uint32_t r0 = atomic_fetch_add(&b->next, PLE_CHUNK);
if (r0 >= b->rows)
break;
uint32_t r1 = r0 + PLE_CHUNK < b->rows ? r0 + PLE_CHUNK : b->rows;
for (uint32_t r = r0; r < r1; r++) {
const float *w = b->w + (size_t)r * b->cols;
uint32_t j = 0;
#if defined(__x86_64__)
for (; v && j + 4 <= b->n; j += 4) {
const float *x[4];
float o[4];
for (int t = 0; t < 4; t++)
x[t] = b->x + (size_t)(j + t) * b->cols;
dot4_f32_avx2(w, x, b->cols, o);
for (int t = 0; t < 4; t++)
b->y[(size_t)(j + t) * b->rows + r] = o[t];
}
#endif
for (; j < b->n; j++)
b->y[(size_t)j * b->rows + r] =
dot_f32(w, b->x + (size_t)j * b->cols, b->cols);
}
}
}
/* y (n rows of t's rows) = t x (n rows of cols): f32 weights as they are,
quantized ones on x put in Q8_K (q, n * cols / 256 blocks) */
static void product(struct janas_llm_model *m, const struct janas_jns_tensor *t,
const float *x, struct janas_block_q8k *q, float *y,
uint32_t n)
{
uint32_t cols = (uint32_t)t->dims[0], rows = (uint32_t)t->dims[1];
if (t->type == 0) {
if (janas_gpu_dense_mm(janas_m_gpu(m, 0), 0, f32(m, t), x, y, rows,
cols, n, 1.0f) == 0)
return;
struct f32_job b = {
.w = f32(m, t), .x = x, .y = y, .rows = rows, .cols = cols, .n = n};
atomic_init(&b.next, 0);
janas_pool_run(m->compute, f32_worker, &b);
return;
}
size_t nb = cols / JANAS_QK;
for (uint32_t j = 0; j < n; j++)
janas_q8k_quantize(x + (size_t)j * cols, q + j * nb, cols);
struct janas_matvec_task tk = {.type = (int)t->type,
.w = data(m, t),
.x = q,
.y = y,
.rows = rows,
.cols = cols,
.n_vec = n};
janas_m_products(m, janas_m_gpu(m, 0), &tk, 1);
}
/*
* The block's per-layer inputs, from the embeddings in x (already scaled by
* sqrt(d_model)): the projection over sqrt(d_model), each layer's part
* normalized, plus the table's row times sqrt(ple_dim), all over sqrt(2).
*/
int janas_m_ple_inputs(struct janas_llm_model *m, const int32_t *tokens,
uint32_t n, uint32_t pos0)
{
uint32_t dm = m->d_model, P = m->ple_dim, W = m->n_layer * P;
float *xb = m->xn; /* the embeddings at bf16 */
for (size_t i = 0; i < (size_t)n * dm; i++)
xb[i] = round_bf16(m->x[i]);
struct bf16_job b = {.w = data(m, m->ple_model),
.x = xb,
.y = m->ple,
.rows = W,
.cols = dm,
.n = n,
.scale = 1.0f / sqrtf((float)dm)};
atomic_init(&b.next, 0);
if (n < 2)
bf16_worker(&b, 0, 1);
else if (janas_gpu_dense_mm(janas_m_gpu(m, 0), 1, b.w, b.x, b.y, b.rows,
b.cols, n, b.scale) != 0)
janas_pool_run(m->compute, bf16_worker, &b);
const float *nw = f32(m, m->ple_norm);
size_t row_bytes =
W / JANAS_QK * janas_qtype_block_size((int)m->ple_tok->type);
uint8_t *raw = (uint8_t *)(m->ple_row + W); /* the row as read */
float es = sqrtf((float)P), is2 = 1.0f / sqrtf(2.0f);
for (uint32_t j = 0; j < n; j++) {
/* a picture's token takes the padding's row (id 0), as both the
reference and llama.cpp do */
int32_t tok =
m->n_media && janas_m_media_row(m, pos0 + j) ? 0 : tokens[j];
off_t at = (off_t)(m->ple_tok->offset + (uint64_t)tok * row_bytes);
const uint8_t *src = raw;
if (m->ple_ram) /* the table in memory (janas_llm_options) */
src = (const uint8_t *)data(m, m->ple_tok) +
(uint64_t)tok * row_bytes;
for (size_t done = 0; !m->ple_ram && done < row_bytes;) {
ssize_t r =
pread(m->j.fd, raw + done, row_bytes - done, at + (off_t)done);
if (r <= 0)
return -1;
done += (size_t)r;
}
janas_dequantize((int)m->ple_tok->type, src, m->ple_row, W);
float *p = m->ple + (size_t)j * W;
for (uint32_t l = 0; l < m->n_layer; l++) {
float *pl = p + (size_t)l * P;
const float *el = m->ple_row + (size_t)l * P;
rms_norm(pl, pl, nw, P, m->eps);
for (uint32_t i = 0; i < P; i++)
pl[i] = (pl[i] + el[i] * es) * is2;
}
}
trace_row(m, "inp_per_layer", m->ple + (size_t)(n - 1) * W, P);
return 0;
}
/*
* Layer l's branch, on the residual in x after the feed-forward, then the
* layer's output scale: x = (x + norm(proj(gelu(gate x) * input))) * s.
*/
void janas_m_ple_layer(struct janas_llm_model *m, const struct layer *ly,
uint32_t l, uint32_t n)
{
uint32_t dm = m->d_model, P = m->ple_dim, W = m->n_layer * P;
product(m, ly->ple_gate, m->x, m->xq, m->ple_g, n);
for (uint32_t j = 0; j < n; j++) {
float *g = m->ple_g + (size_t)j * P;
const float *in = m->ple + (size_t)j * W + (size_t)l * P;
for (uint32_t i = 0; i < P; i++)
g[i] = janas_gelu_mul_det(g[i], in[i]);
}
product(m, ly->ple_proj, m->ple_g, m->ple_q, m->xn, n);
const float *pw = f32(m, ly->ple_post);
for (uint32_t j = 0; j < n; j++) {
float *x = m->x + (size_t)j * dm, *y = m->xn + (size_t)j * dm;
rms_norm(y, y, pw, dm, m->eps);
for (uint32_t i = 0; i < dm; i++)
x[i] = (x[i] + y[i]) * ly->out_scale;
}
trace_row(m, "per_layer_out", m->x + (size_t)(n - 1) * dm, dm);
}