37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
/*
* model.c - Janas-LLM forward pass (see model.h).
*
* What is computed follows the model definitions (llama.cpp's graphs are the
* reference the logits are compared against); how it is computed is Janas's
* own: fused single-pass kernels, a KV cache only for the attention layers,
* the shared expert run as one more expert of the MoE dispatch.
*
* Architectures:
* qwen3moe every layer: attention (q/k norms, NeoX RoPE) + routed experts
* qwen3next every full_attention_interval-th layer: gated attention with
* partial RoPE; the others: Gated DeltaNet (see deltanet.h)
* with a causal convolution; every layer: routed experts plus a
* shared expert scaled by a sigmoid gate
* qwen35moe (Qwen3.5, Qwen3.6) the same layers; the DeltaNet projections
* come separate (q|k|v, z, alpha and beta in f32) instead of
* grouped per key head; the last block of the file is the
* multi-token prediction block, not a layer of the model
* gemma4 sliding-window layers (a 1024-position window) and every
* sixth a full one, each kind with its head size, KV heads
* and RoPE; norms after attention and after the FFN too, GELU,
* a scale per layer, soft-capped logits
* What sets them apart is data, in arch.h; a dense model is a mixture with
* one expert per layer.
*
* This file runs the passes (forward, MTP); the layers are in model_attn.c,
* model_rec.c and model_ffn.c, opening and closing in model_load.c, the
* threads and the experts' profile in model_tune.c, saved states in
* model_state.c.
*/
#define _GNU_SOURCE
#include "model.h"
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include "attention.h"
#include "common/pool.h"
#include "common/sysinfo.h"
#include "deltanet.h"
#include "tuner.h"
#include "gpu.h"
#include "jmath.h"
#include "matvec.h"
#include "quant.h"
#include "vmath.h"
#include "model_impl.h"
struct janas_pool *janas_llm_model_compute(struct janas_llm_model *m)
{
return m->compute;
}
void janas_llm_model_phases(const struct janas_llm_model *m,
double out[JANAS_PH_COUNT])
{
memcpy(out, m->phase, sizeof(m->phase));
/* la sonda stampa e azzera qui: chi legge le fasi sta separando due
tratti (prefill e decodifica), e la scomposizione deve separarli
insieme a lui, altrimenti il prefill sporca la decodifica */
if (janas_m_exp_tracing())
janas_m_exp_report();
}
static void embed(struct janas_llm_model *m, int32_t token, float *x)
{
uint32_t dm = m->d_model;
size_t row_bytes =
dm / JANAS_QK * janas_qtype_block_size((int)m->tok_embd->type);
const uint8_t *row =
(const uint8_t *)data(m, m->tok_embd) + (size_t)token * row_bytes;
janas_dequantize((int)m->tok_embd->type, (const void *)row, x, dm);
if (m->a->sandwich) { /* Gemma: the embedding scaled by sqrt(d_model) */
float s = sqrtf((float)dm);
for (uint32_t i = 0; i < dm; i++)
x[i] *= s;
}
}
void janas_m_embed(struct janas_llm_model *m, int32_t token, float *x)
{
embed(m, token, x);
}
/* RoPE angles of positions pos0 .. pos0 + n - 1 for n_rot dimensions, the
same for every head; ff: a divisor of each pair's angle, or NULL. */
static void rope_table(float *cs, float *sn, uint32_t n, uint32_t pos0,
uint32_t n_rot, float base, const float *ff)
{
uint32_t half = n_rot / 2;
for (uint32_t j = 0; j < n; j++)
for (uint32_t i = 0; i < half; i++) {
float theta = (float)(pos0 + j) *
janas_powf(base, -2.0f * (float)i / (float)n_rot);
if (ff)
theta /= ff[i];
janas_sincosf(theta, &sn[(size_t)j * half + i],
&cs[(size_t)j * half + i]);
}
}
/* With pictures in the sequence: each position's three (time, row, column;
model_media.c), each frequency taking the one the multimodal sections
give it */
static void rope_media(struct janas_llm_model *m, uint32_t n, uint32_t pos0)
{
uint32_t half = m->n_rot / 2;
for (uint32_t j = 0; j < n; j++) {
float p3[3];
janas_m_media_pos(m, pos0 + j, p3);
for (uint32_t i = 0; i < half; i++) {
int axis = i % 3 == 1 && i < 3 * m->mrope[1] ? 1
: i % 3 == 2 && i < 3 * m->mrope[2] ? 2
: 0;
float theta =
p3[axis] *
janas_powf(m->rope_base, -2.0f * (float)i / (float)m->n_rot);
if (m->rope_freqs)
theta /= m->rope_freqs[i];
janas_sincosf(theta, &m->rope_sin[(size_t)j * half + i],
&m->rope_cos[(size_t)j * half + i]);
}
}
}
static void set_rope(struct janas_llm_model *m, uint32_t n, uint32_t pos0)
{
if (m->n_media && (m->mrope[0] || m->mrope[1] || m->mrope[2]))
rope_media(m, n, pos0);
else
rope_table(m->rope_cos, m->rope_sin, n, pos0, m->n_rot, m->rope_base,
m->rope_freqs);
if (m->a->swa)
rope_table(m->rope_cos_swa, m->rope_sin_swa, n, pos0, m->n_rot_swa,
m->rope_base_swa, NULL);
}
/* Gemma 4, after the FFN: x = (xres + rms_norm(x) * post_ffn) * out_scale,
the FFN's output having been summed into a zeroed x */
static void sandwich_worker(void *arg, int tid, int n_threads)
{
struct tok_job *t = arg;
struct janas_llm_model *m = t->m;
uint32_t dm = m->d_model;
for (uint32_t j = (uint32_t)tid; j < t->n; j += (uint32_t)n_threads) {
float *x = m->x + (size_t)j * dm;
const float *r = m->xres + (size_t)j * dm;
rms_norm(x, x, t->w, dm, m->eps);
/* with per-layer embeddings the scale comes after their branch */
float s = m->ple_dim ? 1.0f : t->ly->out_scale;
for (uint32_t i = 0; i < dm; i++)
x[i] = (r[i] + x[i]) * s;
}
}
/* Gemma 4's mixture, token by token: the dense branch's output normalized
aside (xbr), then the experts' input and the router's from the residual,
then the experts' output normalized and the two summed */
static void branch1_worker(void *arg, int tid, int n_threads)
{
struct tok_job *t = arg;
struct janas_llm_model *m = t->m;
uint32_t dm = m->d_model;
for (uint32_t j = (uint32_t)tid; j < t->n; j += (uint32_t)n_threads)
rms_norm(m->xbr + (size_t)j * dm, m->x + (size_t)j * dm,
f32(m, t->ly->post_norm1), dm, m->eps);
}
static void prep2_worker(void *arg, int tid, int n_threads)
{
struct tok_job *t = arg;
struct janas_llm_model *m = t->m;
uint32_t dm = m->d_model;
const float *rs = t->ly->router_scale;
float sd = 1.0f / sqrtf((float)dm);
for (uint32_t j = (uint32_t)tid; j < t->n; j += (uint32_t)n_threads) {
const float *r = m->xres + (size_t)j * dm;
float *xn = m->xn + (size_t)j * dm;
/* the experts' input, to Q8_K through xn */
rms_norm(xn, r, f32(m, t->ly->pre_norm2), dm, m->eps);
janas_q8k_quantize(xn, m->xq + j * (dm / JANAS_QK), dm);
/* the router's: the residual normalized without weights, scaled
by 1 / sqrt(d_model) and per dimension */
double ss = 0.0;
for (uint32_t i = 0; i < dm; i++)
ss += (double)r[i] * r[i];
float sc = 1.0f / sqrtf((float)(ss / dm) + m->eps);
for (uint32_t i = 0; i < dm; i++)
xn[i] = r[i] * sc * sd * rs[i];
}
}
static void branch2_worker(void *arg, int tid, int n_threads)
{
struct tok_job *t = arg;
struct janas_llm_model *m = t->m;
uint32_t dm = m->d_model;
for (uint32_t j = (uint32_t)tid; j < t->n; j += (uint32_t)n_threads) {
float *x = m->x + (size_t)j * dm;
const float *b = m->xbr + (size_t)j * dm;
rms_norm(x, x, f32(m, t->ly->post_norm2), dm, m->eps);
for (uint32_t i = 0; i < dm; i++)
x[i] += b[i];
}
}
static void run_tok(struct janas_llm_model *m, const struct layer *ly,
uint32_t n, void (*fn)(void *, int, int))
{
struct tok_job t = {.m = m, .ly = ly, .n = n};
if (n < 2)
fn(&t, 0, 1);
else
janas_pool_run(m->compute, fn, &t);
}
static int head_of_vocab(struct janas_llm_model *m,
const struct janas_block_q8k *x, float *logits,
uint32_t n);
static int mask_to_vocab(struct janas_llm_model *m, float *logits, uint32_t n);
/* logits soft-capped: c * tanh(l / c) */
static void softcap_worker(void *arg, int tid, int n_threads)
{
struct tok_job *t = arg;
float c = t->m->softcap, *l = t->out;
size_t len = (size_t)t->n * t->m->n_vocab;
size_t per = (len + (size_t)n_threads - 1) / (size_t)n_threads;
size_t a = (size_t)tid * per, b = a + per < len ? a + per : len;
for (size_t i = a; i < b; i++)
l[i] = c * janas_tanhf(l[i] / c);
}
/* A key row turned back by the angles in cs, sn (half pairs, NeoX) and put
in its bits again: a rotation can take a pair past the largest number of
the row, so the scale is found anew. row: int8 or int16 (wide). */
static void unrotate_row(void *row, float *scale, uint32_t hd, uint32_t half,
const float *cs, const float *sn, int wide)
{
float tmp[512], mx = 0.0f, z = *scale;
int8_t *r8 = row;
int16_t *r16 = row;
for (uint32_t i = 0; i < hd; i++)
tmp[i] = (wide ? (float)r16[i] : (float)r8[i]) * z;
for (uint32_t i = 0; i < half; i++) {
float a = tmp[i], b = tmp[i + half];
tmp[i] = a * cs[i] + b * sn[i];
tmp[i + half] = b * cs[i] - a * sn[i];
}
for (uint32_t i = 0; i < hd; i++)
if (fabsf(tmp[i]) > mx)
mx = fabsf(tmp[i]);
float top = wide ? 32767.0f : 127.0f, inv = mx > 0 ? top / mx : 0.0f;
for (uint32_t i = 0; i < hd; i++) {
if (wide)
r16[i] = (int16_t)lrintf(tmp[i] * inv);
else
r8[i] = (int8_t)lrintf(tmp[i] * inv);
}
*scale = mx / top;
}
/*
* Forgets the positions keep .. keep + drop of the attention layers: what
* follows them moves down and its keys are turned back by drop positions,
* since the rotation a key carries depends on where it was. The recurrent
* state is left alone: it is a summary of everything seen, and in a hybrid
* model that is what carries the older text once the window has passed it.
* The first `keep` positions stay where they are, which keeps the system
* message - and the anchor it gives attention - in place. Each slot with its
* own geometry and RoPE (Gemma 4: sliding-window layers and full ones); a
* ring (a sliding-window layer's) keeps only its last positions, and those
* are the ones that move.
*/
int janas_llm_model_shift(struct janas_llm_model *m, uint32_t keep,
uint32_t drop, uint32_t n_kv)
{
/* pictures' positions are not slid (yet): the caller reads the prompt
again instead */
if (!drop || n_kv > m->n_ctx || keep + drop > n_kv || m->hd_max > 512 ||
m->n_media)
return -1;
/* qwen4exp: its n-gram history first (the one part that can refuse) */
if (m->hc_n && janas_m_hc_shift(m, keep, drop, n_kv) != 0)
return -1;
uint32_t move = n_kv - keep - drop, slots = m->n_attn + (m->mtp ? 1u : 0u);
size_t elt = (size_t)1 << m->kv16;
float cs[256], sn[256];
uint8_t *ring_tmp = NULL;
for (uint32_t sl = 0; sl < slots; sl++) {
uint32_t hd = m->kv_hd[sl], cap = m->kv_cap[sl];
int swa = m->kv_swa[sl];
/* Nemotron-H's attention has no RoPE (its n_rot is the buffers'):
the keys move, nothing to turn back */
uint32_t n_rot = m->a->nh ? 0 : swa ? m->n_rot_swa : m->n_rot;
uint32_t half = n_rot / 2;
float base = swa ? m->rope_base_swa : m->rope_base;
const float *ff = swa ? NULL : m->rope_freqs;
for (uint32_t i = 0; i < half; i++) {
float theta =
(float)drop * janas_powf(base, -2.0f * (float)i / (float)n_rot);
if (ff)
theta /= ff[i];
janas_sincosf(theta, &sn[i], &cs[i]);
}
size_t row = hd * elt;
for (uint32_t h = 0; h < m->kv_heads[sl]; h++) {
uint8_t *k = (uint8_t *)m->kcache + m->kv_off[sl] + h * cap * row;
uint8_t *v = (uint8_t *)m->vcache + m->kv_off[sl] + h * cap * row;
float *kz = m->kscale + m->ks_off[sl] + (size_t)h * cap;
float *vz = m->vscale + m->ks_off[sl] + (size_t)h * cap;
if (!m->kv_ring[sl]) {
if (move) {
memmove(k + keep * row, k + (keep + drop) * row,
move * row);
memmove(v + keep * row, v + (keep + drop) * row,
move * row);
memmove(kz + keep, kz + keep + drop, move * sizeof(float));
memmove(vz + keep, vz + keep + drop, move * sizeof(float));
}
for (uint32_t p = keep; p < keep + move && half; p++)
unrotate_row(k + p * row, kz + p, hd, half, cs, sn,
m->kv16);
continue;
}
/* a ring: the positions it still has, from a copy of it, each
to its new row (they cannot land on a kept one: see the
bound on lo) */
if (!ring_tmp &&
!(ring_tmp = malloc(2 * (size_t)cap * (row + sizeof(float)))))
return -1;
uint8_t *tk = ring_tmp, *tv = tk + cap * row;
float *tkz = (float *)(void *)(tv + cap * row), *tvz = tkz + cap;
memcpy(tk, k, cap * row);
memcpy(tv, v, cap * row);
memcpy(tkz, kz, cap * sizeof(float));
memcpy(tvz, vz, cap * sizeof(float));
uint32_t lo = keep + drop;
if (n_kv > cap && n_kv - cap > lo)
lo = n_kv - cap;
for (uint32_t p = lo; p < n_kv; p++) {
uint32_t src = p % cap, dst = (p - drop) % cap;
memcpy(k + dst * row, tk + src * row, row);
memcpy(v + dst * row, tv + src * row, row);
kz[dst] = tkz[src];
vz[dst] = tvz[src];
if (half)
unrotate_row(k + dst * row, kz + dst, hd, half, cs, sn,
m->kv16);
}
}
}
free(ring_tmp);
if (m->hc_n) /* and the indexer's keys and blocks */
janas_m_qsa_shift(m, keep, drop, n_kv);
if (m->ring_hi > keep + drop)
m->ring_hi -= drop;
/* the recurrent state holds what it has seen, dropped tokens included -
that is what carries the older text - but the position it stands at
moves with everything else */
if (m->base_pos != UINT32_MAX) {
uint32_t b = m->base_pos, t = b + m->log_n;
if (b >= keep + drop)
m->base_pos = b - drop;
else if (m->log_n && t > keep) {
/* the log runs into what is dropped (or past it): replaying
it would count the dropped tokens and land short of the
tip, so the state goes on from the tip, the other buffer */
m->base = 1 - m->base;
m->base_pos = t >= keep + drop ? t - drop : keep;
m->log_n = 0;
} else if (b > keep)
m->base_pos = keep;
}
return 0;
}
static int forward(struct janas_llm_model *m, const int32_t *tokens, uint32_t n,
uint32_t pos0, float *logits, int all_logits);
/*
* Reading ahead where it pays: it costs a router and the reads' submission
* on every token (~4 ms on Qwen3-Next-80B), and gains only where the cache
* misses many experts. Measured on 30 Sep 2026 (that model, every bit): -3%
* at ~12 misses a token, even at ~40, +4.5% at ~77, +5-7.5% at ~145. So on
* above PF_ON experts a token the cache would have missed (its misses and
* those the reads ahead spared it), off below PF_OFF, as it was between
* them; off, it costs nothing.
*/
#define PF_WINDOW 32
#define PF_ON 60
#define PF_OFF 40
/*
* Reading ahead in prefill blocks (unless JANAS_PREFETCH_PREFILL=0 or 2):
* on after a block whose layers would have missed more than PFB_ON experts
* each, as a rule (the misses and those the reads ahead spared); only for
* blocks of PFB_MIN_N tokens or more, the passes that verify drafts being
* decoding. It costs layer l + 1's router once more on every token (1.1
* ms a token on Qwen3-Next-80B) and spares half the time waited for the
* disk where the cache is small for the model (6 Oct 2026).
*/
#define PFB_MIN_N 32
#define PFB_ON 32
static void prefetch_block_decide(struct janas_llm_model *m,
const struct janas_expert_cache_stats *a,
const struct janas_expert_cache_stats *b)
{
uint64_t would =
(b->misses - a->misses) + (b->prefetch_used - a->prefetch_used);
m->pfb_on = would > (uint64_t)PFB_ON * m->n_layer;
}
static void prefetch_decide(struct janas_llm_model *m,
const struct janas_expert_cache_stats *a,
const struct janas_expert_cache_stats *b)
{
m->pf_would +=
(b->misses - a->misses) + (b->prefetch_used - a->prefetch_used);
if (++m->pf_tokens < PF_WINDOW)
return;
uint64_t per = m->pf_would / m->pf_tokens;
if (!m->pf_on && per > PF_ON)
m->pf_on = 1;
else if (m->pf_on && per < PF_OFF)
m->pf_on = 0;
m->pf_tokens = 0;
m->pf_would = 0;
}
/*
* A prompt's blocks read with the GPU's whole pass or without it (the
* CPU's attention, the products where they go), whichever this machine
* reads faster at this position: the attention's cost grows with the
* position, and on the GPU it may grow faster or slower than on the CPU -
* Qwen3-4B on the development laptop's Arc read 4,096 tokens at 72.6
* tokens/s with it and 95.9 without, 16,384 at 31.8 and 55.6 (26 Oct
* 2026), where a card with fast memory of its own may well win. So every
* LP_BLOCKS of the engine's blocks of positions one block each way,
* measured in turn (the first of a band alternating: a later block costs
* more), and the faster for the rest of the band - no threshold, only how
* often to look again, at most one slower block in LP_BLOCKS.
* JANAS_GPU_LONG=0: the whole pass always, as before.
*/
#define LP_BLOCKS 8
static int long_prompt_on(const struct janas_llm_model *m, uint32_t n,
uint32_t pos0)
{
static int off = -1;
if (off < 0) {
const char *e = getenv("JANAS_GPU_LONG");
off = e && atoi(e) <= 0;
}
(void)pos0;
/* the whole pass asked for (JANAS_GPU_TOKEN=1): as asked */
return !off && n >= JANAS_GPU_FAST_N && m->gtok_state == 1 &&
janas_llm_model_gpu_prompts(m) && !janas_m_gpu_token_asked();
}
/* this block's way: lp_cpu set for the one without the whole pass */
static void long_prompt_begin(struct janas_llm_model *m, uint32_t pos0)
{
uint32_t band = pos0 / (LP_BLOCKS * m->blk);
if (band != m->lp.band || m->lp.state == 0) {
if (band != m->lp.band)
m->lp.first = (int)(band & 1u);
m->lp.band = band;
m->lp.state = 0;
}
if (m->lp.state < 2) /* the first way, then the other */
m->lp_cpu = m->lp.state == 0 ? m->lp.first : !m->lp.first;
else
m->lp_cpu = m->lp.cpu < m->lp.gpu;
}
static void long_prompt_end(struct janas_llm_model *m, double sec_per_token,
int on_gpu)
{
int cpu = m->lp_cpu;
m->lp_cpu = 0;
if (m->lp.state >= 2 || sec_per_token <= 0 || on_gpu == cpu)
return; /* chosen already; or not the way meant (none measured) */
*(cpu ? &m->lp.cpu : &m->lp.gpu) = sec_per_token;
if (++m->lp.state == 2 && getenv("JANAS_GPU_VERBOSE"))
fprintf(stderr,
"janas: a prompt from position %u: %.2f ms a token with the "
"GPU's whole pass, %.2f without; %s\n",
m->lp.band * LP_BLOCKS * m->blk, m->lp.gpu * 1e3,
m->lp.cpu * 1e3, m->lp.cpu < m->lp.gpu ? "without" : "with it");
}
int janas_llm_model_forward(struct janas_llm_model *m, const int32_t *tokens,
uint32_t n, uint32_t pos0, float *logits,
int all_logits)
{
int lp = long_prompt_on(m, n, pos0);
uint64_t runs0 = m->gtok_runs[2];
if (lp)
long_prompt_begin(m, pos0);
/* each pass measured for the tuner: the configuration never changes
the results */
int cls = janas_tune_class(n), cand = janas_tuner_pick(&m->tuner, cls);
janas_m_apply_candidate(m, cand, n);
/* measured without the time spent waiting for experts read from disk:
it depends on the cache, not on the configuration */
struct janas_expert_cache_stats c0, c1;
janas_expert_cache_stats(m->cache, &c0);
double k0 = janas_llm_model_npu_compile_seconds(m), t0 = now();
int r = forward(m, tokens, n, pos0, logits, all_logits);
double dt = now() - t0;
janas_m_npu_pass_end(m);
janas_expert_cache_stats(m->cache, &c1);
dt -= c1.wait_seconds - c0.wait_seconds;
/* nor the NPU's first compilation of a matrix: once a session */
dt -= janas_llm_model_npu_compile_seconds(m) - k0;
m->last_pass_s = dt > 0 && !m->tuner.trying ? dt : 0;
/* while the cache is still filling in the background, passes are not
representative: neither the tuner nor the draft planner may learn
from them, or a slow start decides the whole session */
if (lp)
long_prompt_end(m, r == 0 && dt > 0 ? dt / n : 0,
m->gtok_runs[2] > runs0);
if (r == 0 && dt > 0 && !janas_llm_model_settling(m))
janas_tuner_record(&m->tuner, cls, cand, dt / n);
if (r == 0 && n == 1 && m->pf_mode == 1)
prefetch_decide(m, &c0, &c1);
if (r == 0 && n >= PFB_MIN_N && m->pfb_mode == 1)
prefetch_block_decide(m, &c0, &c1);
janas_m_arena_to_gpu(m);
return r;
}
double janas_llm_model_pass_seconds(const struct janas_llm_model *m)
{
return m->last_pass_s;
}
static int forward(struct janas_llm_model *m, const int32_t *tokens, uint32_t n,
uint32_t pos0, float *logits, int all_logits)
{
if (n < 1 || n > m->blk || pos0 + n > m->n_ctx)
return -1;
for (uint32_t j = 0; j < n; j++)
if (tokens[j] < 0 || (uint32_t)tokens[j] >= m->n_vocab)
return -1;
uint32_t dm = m->d_model;
size_t nbd = dm / JANAS_QK;
for (uint32_t j = 0; j < n; j++) {
/* a picture's token: its embedding from the encoder */
const float *pic = m->n_media ? janas_m_media_row(m, pos0 + j) : NULL;
if (pic)
memcpy(m->x + (size_t)j * dm, pic, sizeof(float) * dm);
else
embed(m, tokens[j], m->x + (size_t)j * dm);
}
if (m->trace >= 2) {
fprintf(stderr, "tokens:");
for (uint32_t j = 0; j < n; j++)
fprintf(stderr, " %d", tokens[j]);
fprintf(stderr, "\n");
trace_row(m, "embed", m->x + (size_t)(n - 1) * dm, dm);
}
/* what can fail before the state is touched: a failure leaves it as
it was */
if (m->ple_dim && janas_m_ple_inputs(m, tokens, n, pos0) != 0)
return -1;
/* from the start: nothing of the GPU's copy of the state matters, the
host's (the recurrent states zeroed below) is the one */
if (m->gtok_dev && pos0 == 0)
m->gtok_where = JANAS_GT_HOST;
if (m->a->rec) {
if (pos0 == 0) {
m->src = 0;
m->replay = 0;
memset(m->ssm_buf, 0,
(size_t)m->n_rec * m->ssm_per * sizeof(uint16_t));
memset(m->conv_buf, 0,
(size_t)m->n_rec * m->conv_ch * (m->d_conv - 1) *
sizeof(float));
} else if (m->base_pos == UINT32_MAX || pos0 < m->base_pos ||
pos0 > m->base_pos + m->log_n) {
return -1; /* before the last call: that state is gone */
} else if (pos0 == m->base_pos + m->log_n && m->log_n > 0) {
m->src = 1 - m->base; /* from the tip */
m->replay = 0;
} else {
m->src = m->base; /* replay the accepted tokens on the base */
m->replay = pos0 - m->base_pos;
}
m->base_pos = UINT32_MAX; /* invalid until this pass completes */
}
set_rope(m, n, pos0);
/* a token (or, with JANAS_GPU_BLOCK=1, a block) on the GPU, whole,
where it can (model_gpu.c) */
/* not where the NPU takes this pass's products: then the block goes
product by product, the NPU's to it, the others as before */
m->gtok_tokens = tokens; /* qwen4exp's n-gram embeddings */
int gpu = logits && !janas_m_npu_takes(m, n)
? janas_m_gpu_token(m, pos0, n, all_logits, logits)
: -1;
if (gpu == -2)
return -1; /* the GPU failed after replaying on the base in place:
no state to go on from (base_pos stays invalid) */
if (gpu != 0) /* the CPU's pass: the state the GPU made newer back */
janas_m_gtok_host(m, 1);
if (gpu == 0) {
/* Gemma 4's soft cap, as below */
if (m->softcap > 0.0f) {
struct tok_job sc = {
.m = m, .n = all_logits ? n : 1, .out = logits};
janas_pool_run(m->compute, softcap_worker, &sc);
}
/* its head the whole vocabulary's: the others -inf here */
if (m->vs_n && mask_to_vocab(m, logits, all_logits ? n : 1) != 0)
return -1;
m->ring_hi = pos0 + n;
if (m->a->rec && n <= JANAS_LLM_MAX_BLOCK) {
/* replayed and logged by the GPU: as at the end */
m->base = m->src;
m->base_pos = pos0;
m->log_n = n;
} else if (m->a->rec) { /* a long block: as at the end */
m->base = 1 - m->src;
m->base_pos = pos0 + n;
m->log_n = 0;
}
return 0;
}
/* Nemotron-H: one part a layer (model_nh.c) */
if (m->a->nh) {
if (janas_m_nh_layers(m, n, pos0) != 0)
return -1;
goto output;
}
/* qwen4exp: its layers on the streams of the residual (model_hc.c) */
if (m->hc_n) {
if (janas_m_hc_layers(m, tokens, n, pos0) != 0)
return -1;
goto output;
}
/* a dense block's first layers on the NPU, graph by graph, as many as
it holds (model_npu.c); the others here */
int npu_layers = janas_m_npu_layers(m, n, pos0);
if (npu_layers == -2)
return -1; /* the NPU failed halfway: no state to go on from */
for (uint32_t l = npu_layers > 0 ? (uint32_t)npu_layers : 0; l < m->n_layer;
l++) {
m->ana_layer = l; /* the Ana-Net probe's row */
const struct layer *ly = &m->layers[l];
janas_m_norm_quant(m, f32(m, ly->attn_norm), 0, n);
if (m->npu) /* the NPU's calibration (JANAS_NPU_A8=1) */
janas_m_npu_calib(m, 3, l, m->xn, n, dm, dm);
trace_row(m, "attn_norm", m->xn + (size_t)(n - 1) * dm, dm);
if (ly->rec)
janas_m_rec_layer(m, ly, n);
else
janas_m_attn_layer(m, ly, n, pos0);
/* the early predictor: layer l + 1's router on the state before
layer l's experts (its norm and router buffers are made again
just below) */
if (m->pred_fn && m->pred_early && l + 1 < m->n_layer &&
janas_m_route_predict(m, l + 1, n) != 0)
return -1;
if (m->pf_on && n == 1 && l + 1 < m->n_layer)
janas_m_prefetch(m, l);
else if (m->pfb_on && n >= PFB_MIN_N && l + 1 < m->n_layer)
janas_m_prefetch_block(m, l, n);
janas_m_norm_quant(m, f32(m, ly->ffn_norm), 0, n);
if (m->npu)
janas_m_npu_calib(m, 1, l, m->xn, n, dm, dm);
trace_row(m, "post_norm", m->xn + (size_t)(n - 1) * dm, dm);
if (ly->post_ffn) {
/* the FFN's output normalized before the residual: summed
into a zeroed x, the residual aside */
memcpy(m->xres, m->x, (size_t)n * dm * sizeof(float));
memset(m->x, 0, (size_t)n * dm * sizeof(float));
}
janas_m_janet_rec(m, l, n, 0); /* the Janet probe (model_janet.c) */
if (janas_m_janet_ffn(m, l, n)) {
/* Janet's detectors in place of the layer's feed-forward */
} else if (ly->pre_norm2) {
/* Gemma 4's mixture: the dense branch on xn / xq as normed
above, then the experts' branch */
size_t bytes = (size_t)n * dm * sizeof(float);
if (janas_m_moe_block(m, ly, m->cache, &m->j.layers[l], l, n,
MOE_SHARED) != 0)
return -1;
run_tok(m, ly, n, branch1_worker);
trace_row(m, "ffn_mlp", m->xbr + (size_t)(n - 1) * dm, dm);
run_tok(m, ly, n, prep2_worker);
memset(m->x, 0, bytes);
if (janas_m_moe_block(m, ly, m->cache, &m->j.layers[l], l, n,
MOE_ROUTED) != 0)
return -1;
run_tok(m, ly, n, branch2_worker);
trace_row(m, "ffn_moe_comb", m->x + (size_t)(n - 1) * dm, dm);
} else if (janas_m_moe_block(m, ly, m->cache, &m->j.layers[l], l, n,
MOE_ALL) != 0) {
return -1;
}
janas_m_janet_rec(m, l, n, 1);
if (ly->post_ffn) {
struct tok_job st = {
.m = m, .ly = ly, .w = f32(m, ly->post_ffn), .n = n};
if (n < 2)
sandwich_worker(&st, 0, 1);
else
janas_pool_run(m->compute, sandwich_worker, &st);
}
if (m->ple_dim)
janas_m_ple_layer(m, ly, l, n);
if (m->pred_fn && !m->pred_early && l + 1 < m->n_layer &&
janas_m_route_predict(m, l + 1, n) != 0)
return -1;
if (m->trace) {
/* JANAS_TRACE=1: the state after each layer, last token of the
block, to compare with another implementation */
const float *v = m->x + (size_t)(n - 1) * dm;
double nn = 0;
for (uint32_t i = 0; i < dm; i++)
nn += (double)v[i] * v[i];
fprintf(stderr, "l_out-%-3u norm %10.4f %9.4f %9.4f %9.4f %9.4f\n",
l, sqrt(nn), v[0], v[1], v[2], v[3]);
}
}
janas_m_ana_frame(m, n); /* the Ana-Net probe's picture */
output:
m->ring_hi = pos0 + n; /* the rings hold up to here */
double td = now();
uint32_t first = all_logits ? 0 : n - 1;
/* with an MTP block, every token's final hidden state is kept for it:
qwen4exp's is its residual's streams before the last mix, which is
the output norm (llama.cpp's h_nextn) */
int keep = m->mtp || m->asst;
if (m->hc_n && keep)
memcpy(m->hid, m->hc_res, (size_t)n * m->hc_n * dm * sizeof(float));
if (m->hc_n) {
janas_m_hc_head(m, first, n);
td = now(); /* the mix's time is the mixes' */
} else
janas_m_norm_quant(m, f32(m, m->out_norm), keep ? 0 : first, n);
if (keep && !m->hc_n)
memcpy(m->hid, m->xn, (size_t)n * dm * sizeof(float));
if (logits && m->vs_n) { /* a vocabulary of its own: its rows alone */
if (head_of_vocab(m, m->xq + first * nbd, logits, n - first) != 0)
return -1;
} else if (logits) { /* without: the hidden states alone, an embedding */
struct janas_matvec_task to = {.type = (int)m->output->type,
.w = data(m, m->output),
.x = m->xq + first * nbd,
.y = logits,
.rows = m->n_vocab,
.cols = dm,
.n_vec = n - first};
janas_m_products(m, janas_m_gpu(m, 0), &to, 1);
}
if (logits && m->softcap > 0.0f && !m->vs_n) {
struct tok_job sc = {.m = m, .n = n - first, .out = logits};
janas_pool_run(m->compute, softcap_worker, &sc);
}
m->phase[JANAS_PH_OUTPUT] += now() - td;
if (m->a->rec && n <= JANAS_LLM_MAX_BLOCK) {
m->base = m->src;
m->base_pos = pos0;
m->log_n = n;
} else if (m->a->rec) {
/* not logged: the state stands at the end of the call, in the
buffer the call wrote, and nothing before it can be replayed */
m->base = 1 - m->src;
m->base_pos = pos0 + n;
m->log_n = 0;
}
return 0;
}
uint32_t janas_llm_model_max_block(const struct janas_llm_model *m)
{
return m->blk;
}
int janas_llm_model_has_mtp(const struct janas_llm_model *m)
{
return m->mtp || m->asst;
}
int janas_llm_mtp_chain_fixed(const struct janas_llm_model *m)
{
return m->asst != NULL;
}
int janas_llm_model_recurrent(const struct janas_llm_model *m)
{
return m->a->rec != JANAS_REC_NONE;
}
const float *janas_llm_model_hidden(const struct janas_llm_model *m)
{
return m->hid;
}
uint32_t janas_llm_model_hidden_width(const struct janas_llm_model *m)
{
return m->hc_n ? m->hc_n * m->d_model : m->d_model;
}
const float *janas_llm_model_normed(const struct janas_llm_model *m)
{
return m->xn;
}
const float *janas_llm_mtp_hidden(const struct janas_llm_model *m)
{
return m->mtp_hid;
}
#if defined(__x86_64__)
/* Sum of exp(l[i] - mx) over whole groups of 8; *done: how many summed. */
__attribute__((target("avx2,fma"))) static double
exp_sum_avx2(const float *l, uint32_t n, float mx, uint32_t *done)
{
__m256 acc = _mm256_setzero_ps(), vm = _mm256_set1_ps(mx);
uint32_t i = 0;
for (; i + 8 <= n; i += 8)
acc = _mm256_add_ps(
acc, janas_exp8(_mm256_sub_ps(_mm256_loadu_ps(l + i), vm)));
*done = i;
return janas_hsum8(acc);
}
#endif
double janas_m_exp_sum(const float *l, uint32_t n, float mx)
{
double sum = 0;
uint32_t i = 0;
#if defined(__x86_64__)
if (__builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma"))
sum = exp_sum_avx2(l, n, mx, &i);
#endif
for (; i < n; i++)
sum += janas_exp_det(l[i] - mx);
return sum;
}
int janas_llm_model_set_vocab(struct janas_llm_model *m, const int32_t *ids,
uint32_t n)
{
m->vs_n = m->vs_cap = 0;
if (!n)
return 0;
int type = (int)m->output->type;
if (type == JANAS_Q6_K_P || !janas_qtype_block_size(type))
return -1;
size_t row = m->d_model / JANAS_QK * janas_qtype_block_size(type);
uint8_t *w = realloc(m->vs_w, (size_t)n * row);
if (!w)
return -1;
m->vs_w = w;
int32_t *v = realloc(m->vs_ids, n * sizeof(int32_t));
if (!v)
return -1;
m->vs_ids = v;
const uint8_t *src = data(m, m->output);
uint32_t k = 0;
for (uint32_t i = 0; i < n; i++) {
if (ids[i] < 0 || (uint32_t)ids[i] >= m->n_vocab)
continue;
memcpy(m->vs_w + (size_t)k * row, src + (size_t)ids[i] * row, row);
m->vs_ids[k++] = ids[i];
}
m->vs_n = k;
return 0;
}
/* The logits of n vectors through the vocabulary's rows alone, the others
-inf (janas_llm_model_set_vocab). */
static int vocab_room(struct janas_llm_model *m, uint32_t n)
{
if (n > m->vs_cap) {
float *y = realloc(m->vs_y, (size_t)n * m->vs_n * sizeof(float));
if (!y)
return -1;
m->vs_y = y;
m->vs_cap = n;
}
return 0;
}
/* The vocabulary's logits of n rows into vs_y, then every row -inf but
for them. */
static void scatter_vocab(struct janas_llm_model *m, float *logits, uint32_t n)
{
for (uint32_t j = 0; j < n; j++) {
float *l = logits + (size_t)j * m->n_vocab;
for (uint32_t i = 0; i < m->n_vocab; i++)
l[i] = -INFINITY;
for (uint32_t i = 0; i < m->vs_n; i++)
l[m->vs_ids[i]] = m->vs_y[(size_t)j * m->vs_n + i];
}
}
/* Logits of the whole head (the GPU's whole pass) cut to the vocabulary. */
static int mask_to_vocab(struct janas_llm_model *m, float *logits, uint32_t n)
{
if (vocab_room(m, n) != 0)
return -1;
for (uint32_t j = 0; j < n; j++)
for (uint32_t i = 0; i < m->vs_n; i++)
m->vs_y[(size_t)j * m->vs_n + i] =
logits[(size_t)j * m->n_vocab + m->vs_ids[i]];
scatter_vocab(m, logits, n);
return 0;
}
static int head_of_vocab(struct janas_llm_model *m,
const struct janas_block_q8k *x, float *logits,
uint32_t n)
{
if (vocab_room(m, n) != 0)
return -1;
struct janas_matvec_task t = {.type = (int)m->output->type,
.w = m->vs_w,
.x = x,
.y = m->vs_y,
.rows = m->vs_n,
.cols = m->d_model,
.n_vec = n};
/* on the CPU: the GPU and the NPU know the model's weights, not these */
janas_matvec_q4k_group(m->compute, &t, 1);
if (m->softcap > 0.0f) /* as softcap_worker, on the rows computed */
for (size_t i = 0; i < (size_t)n * m->vs_n; i++)
m->vs_y[i] = m->softcap * janas_tanhf(m->vs_y[i] / m->softcap);
scatter_vocab(m, logits, n);
return 0;
}
void janas_llm_mtp_note_tokens(struct janas_llm_model *m, const int32_t *tokens,
uint32_t n)
{
if (!m->dh_w)
return;
for (uint32_t j = 0; j < n; j++) {
int32_t t = tokens[j];
if (t < 0 || (uint32_t)t >= m->n_vocab || m->dh_in[t] ||
m->dh_n == m->dh_cap)
continue;
memcpy(m->dh_w + (size_t)m->dh_n * m->dh_row,
m->dh_src + (size_t)t * m->dh_row, m->dh_row);
m->dh_ids[m->dh_n++] = t;
m->dh_in[t] = 1;
}
}
int janas_llm_mtp_draft(struct janas_llm_model *m, const int32_t *tokens,
const float *hidden, uint32_t n, uint32_t pos0,
int32_t *token, float *conf)
{
if (m->asst) { /* Gemma 4's assistant: the last row alone */
uint32_t dm = m->d_model;
if (n < 1 || n > m->blk)
return -1;
return janas_m_assist_draft(
m, tokens[n - 1], hidden + (size_t)(n - 1) * dm, pos0 + n - 1,
m->mtp_hid + (size_t)(n - 1) * dm, token, conf);
}
if (janas_llm_mtp_forward(m, tokens, hidden, n, pos0, NULL, 0) != 0)
return -1;
janas_m_apply_candidate(m, janas_tuner_best(&m->tuner, 0),
1); /* one-row head */
double td = now();
size_t nbd = m->d_model / JANAS_QK;
struct janas_matvec_task t = {.type = (int)m->output->type,
.w = m->dh_w,
.x = m->xq + (n - 1) * nbd,
.y = m->dh_logits,
.rows = m->dh_n,
.cols = m->d_model};
janas_m_products(m, janas_m_gpu(m, 0), &t, 1);
/* the best row and its softmax probability within the head */
float *l = m->dh_logits, mx = l[0];
uint32_t best = 0;
for (uint32_t i = 1; i < m->dh_n; i++)
if (l[i] > mx) {
mx = l[i];
best = i;
}
double sum = 0;
uint32_t i = 0;
#if defined(__x86_64__)
if (m->avx2)
sum = exp_sum_avx2(l, m->dh_n, mx, &i);
#endif
for (; i < m->dh_n; i++)
sum += janas_exp_det(l[i] - mx);
*token = m->dh_ids[best];
*conf = (float)(1.0 / sum);
m->phase[JANAS_PH_OUTPUT] += now() - td;
return 0;
}
int janas_llm_mtp_forward(struct janas_llm_model *m, const int32_t *tokens,
const float *hidden, uint32_t n, uint32_t pos0,
float *logits, int all_logits)
{
if (m->asst) /* no keys and values of its own: nothing to fill */
return logits || n < 1 || n > m->blk ? -1 : 0;
if (!m->mtp || n < 1 || n > m->blk || pos0 + n > m->n_ctx)
return -1;
for (uint32_t j = 0; j < n; j++)
if (tokens[j] < 0 || (uint32_t)tokens[j] >= m->n_vocab)
return -1;
janas_m_apply_candidate(m, janas_tuner_best(&m->tuner, janas_tune_class(n)),
n);
uint32_t dm = m->d_model;
size_t nbd = dm / JANAS_QK, nbc = 2 * dm / JANAS_QK;
const struct layer *ly = &m->mtp_layer;
double ta = now();
if (m->hc_n) { /* qwen4exp's block, on the residual's streams */
uint32_t first = all_logits ? 0 : n - 1;
for (uint32_t j = 0; j < n; j++)
embed(m, tokens[j], m->x + (size_t)j * dm);
set_rope(m, n, pos0);
if (janas_m_hc_mtp(m, hidden, n, pos0, first) != 0)
return -1;
m->phase[JANAS_PH_QKV] += now() - ta;
if (!logits)
return 0;
double tl = now();
struct janas_matvec_task to = {.type = (int)m->output->type,
.w = data(m, m->output),
.x = m->xq + first * nbd,
.y = logits,
.rows = m->n_vocab,
.cols = dm,
.n_vec = n - first};
janas_m_products(m, janas_m_gpu(m, 0), &to, 1);
m->phase[JANAS_PH_OUTPUT] += now() - tl;
return 0;
}
/*
* [norm(embedding); norm(hidden)], projected back to d_model. Settled on
* data (tests/mtp_eval.c, 398 positions of C source): this order, with
* the hidden state after the main model's final norm, matches the main
* model's next choice 88.9% of the times; the state before the norm
* 86.4%; the other order 0%.
*/
for (uint32_t j = 0; j < n; j++) {
float *c = m->cat + (size_t)j * 2 * dm;
embed(m, tokens[j], m->x + (size_t)j * dm);
rms_norm(c, m->x + (size_t)j * dm, f32(m, m->enorm), dm, m->eps);
rms_norm(c + dm, hidden + (size_t)j * dm, f32(m, m->hnorm), dm, m->eps);
janas_q8k_quantize(c, m->catq + j * nbc, 2 * dm);
}
struct janas_matvec_task tp = {.type = (int)m->eh_proj->type,
.w = data(m, m->eh_proj),
.x = m->catq,
.y = m->x,
.rows = dm,
.cols = 2 * dm,
.n_vec = n};
janas_m_products(m, janas_m_gpu(m, 0), &tp, 1);
m->phase[JANAS_PH_QKV] += now() - ta;
set_rope(m, n, pos0);
for (uint32_t j = 0; j < n; j++) {
rms_norm(m->xn + (size_t)j * dm, m->x + (size_t)j * dm,
f32(m, ly->attn_norm), dm, m->eps);
janas_q8k_quantize(m->xn + (size_t)j * dm, m->xq + j * nbd, dm);
}
janas_m_attn_layer(m, ly, n, pos0);
for (uint32_t j = 0; j < n; j++) {
rms_norm(m->xn + (size_t)j * dm, m->x + (size_t)j * dm,
f32(m, ly->ffn_norm), dm, m->eps);
janas_q8k_quantize(m->xn + (size_t)j * dm, m->xq + j * nbd, dm);
}
if (janas_m_moe_block(m, ly, m->mtp_cache, m->mtp_jl, m->mtp_cl, n,
MOE_ALL) != 0)
return -1;
double td = now();
uint32_t first = all_logits ? 0 : n - 1;
for (uint32_t j = 0; j < n; j++) {
rms_norm(m->xn + (size_t)j * dm, m->x + (size_t)j * dm,
f32(m, m->mtp_norm), dm, m->eps);
janas_q8k_quantize(m->xn + (size_t)j * dm, m->xq + j * nbd, dm);
}
memcpy(m->mtp_hid, m->xn, (size_t)n * dm * sizeof(float));
if (!logits) { /* only the KV cache and the hidden states wanted */
m->phase[JANAS_PH_OUTPUT] += now() - td;
return 0;
}
struct janas_matvec_task to = {.type = (int)m->output->type,
.w = data(m, m->output),
.x = m->xq + first * nbd,
.y = logits,
.rows = m->n_vocab,
.cols = dm,
.n_vec = n - first};
janas_m_products(m, janas_m_gpu(m, 0), &to, 1);
m->phase[JANAS_PH_OUTPUT] += now() - td;
return 0;
}
int janas_llm_model_decode(struct janas_llm_model *m, int32_t token,
uint32_t pos, float *logits)
{
return janas_llm_model_forward(m, &token, 1, pos, logits, 0);
}