37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
/*
* model_load.c - opening a model: its tensors layer by layer, the MTP
* block, the buffers, and closing it (see model.h).
*/
#define _GNU_SOURCE
#include "model.h"
#include "janas/llm.h" /* JANAS_LLM_QUALITY_* */
#include <errno.h>
#include <math.h>
#include <pthread.h>
#include <sched.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <time.h>
#include <unistd.h>
#include "attention.h"
#include "common/aligned.h"
#include "common/pool.h"
#include "common/sysinfo.h"
#include "deltanet.h"
#include "tuner.h"
#include "gpu.h"
#include "matvec.h"
#include "quant.h"
#include "vmath.h"
#include "model_impl.h"
static int meta_u32(struct janas_llm_model *m, const char *suffix, uint32_t *v)
{
char key[128];
snprintf(key, sizeof(key), "%.32s.%s", m->j.h.arch, suffix);
return janas_jns_meta_u32(&m->j, key, v);
}
static int meta_f32(struct janas_llm_model *m, const char *suffix, float *v)
{
char key[128];
snprintf(key, sizeof(key), "%.32s.%s", m->j.h.arch, suffix);
return janas_jns_meta_f32(&m->j, key, v);
}
enum { T_QUANT, T_F32, T_F16 };
/* Tensor by name with an expected type class and shape (d1 = 0: any). */
static const struct janas_jns_tensor *need(const struct janas_jns *j,
const char *name, int kind,
uint64_t d0, uint64_t d1, char *err,
size_t err_len)
{
const struct janas_jns_tensor *t = janas_jns_tensor(j, name);
if (!t) {
snprintf(err, err_len, "tensor %s missing", name);
return NULL;
}
int ok = t->dims[0] == d0 && (d1 == 0 || t->dims[1] == d1);
if (kind == T_F32)
ok = ok && t->type == 0;
else if (kind == T_F16)
ok = ok && t->type == 1;
else /* the bit planes are the experts' alone: a resident tensor in
them could be neither unpacked (janas_dequantize) nor read */
ok = ok && janas_qtype_block_size((int)t->type) != 0 &&
t->type != JANAS_Q6_K_P && t->type != JANAS_Q6_K_PN;
if (!ok) {
snprintf(err, err_len, "tensor %s: unexpected type or shape", name);
return NULL;
}
return t;
}
/* Hyperparameters of the Gated DeltaNet layers. */
static int load_ssm_params(struct janas_llm_model *m)
{
if (meta_u32(m, "ssm.conv_kernel", &m->d_conv) ||
meta_u32(m, "ssm.state_size", &m->ds) ||
meta_u32(m, "ssm.group_count", &m->n_kh) ||
meta_u32(m, "ssm.time_step_rank", &m->n_vh) ||
meta_u32(m, "ssm.inner_size", &m->d_inner))
return -1;
if (m->d_conv < 2 || m->d_conv > MAX_CONV || m->ds % SSM_ROWS ||
m->ds > MAX_SSM_DIM || m->n_kh == 0 || m->n_vh % m->n_kh ||
m->d_inner != m->n_vh * m->ds || m->d_inner % JANAS_QK)
return -1;
m->conv_ch = 2 * m->n_kh * m->ds + m->d_inner;
m->qkvz_dim = 2 * m->n_kh * m->ds + 2 * m->d_inner;
m->ssm_per = (size_t)m->n_vh * m->ds * m->ds;
return 0;
}
/*
* Gemma 4: sliding-window layers and full ones, each kind with its head
* size, RoPE and KV heads; the KV heads come as an array, one per layer,
* and have to be the same within a kind (one attention per kind). *pat:
* the pattern, one byte per layer, nonzero for a sliding-window layer.
*/
static int load_swa_params(struct janas_llm_model *m, const uint8_t **pat)
{
char key[128];
uint64_t np = 0, nk = 0;
const int32_t *kv = NULL;
snprintf(key, sizeof(key), "%.32s.attention.sliding_window_pattern",
m->j.h.arch);
if (janas_jns_meta_bytes(&m->j, key, &np, pat) || np < m->n_layer)
return -1;
snprintf(key, sizeof(key), "%.32s.attention.head_count_kv", m->j.h.arch);
/* E2B and E4B give one count for both kinds */
uint32_t one = 0;
if (janas_jns_meta_ints(&m->j, key, &nk, &kv) || nk < m->n_layer) {
if (janas_jns_meta_u32(&m->j, key, &one) || one == 0)
return -1;
kv = NULL;
}
m->n_head_kv = m->n_kv_swa = 0;
for (uint32_t l = 0; l < m->n_layer; l++) {
int32_t h = (int32_t)one;
if (kv)
memcpy(&h, kv + l, 4);
uint32_t *want = (*pat)[l] ? &m->n_kv_swa : &m->n_head_kv;
if (h <= 0 || (*want && *want != (uint32_t)h))
return -1;
*want = (uint32_t)h;
}
m->softcap = 0.0f;
meta_f32(m, "final_logit_softcapping", &m->softcap);
if (m->n_head_kv == 0 || m->n_kv_swa == 0 ||
meta_u32(m, "attention.key_length_swa", &m->head_swa) ||
meta_u32(m, "rope.dimension_count_swa", &m->n_rot_swa) ||
meta_f32(m, "rope.freq_base_swa", &m->rope_base_swa) ||
meta_u32(m, "attention.sliding_window", &m->window) ||
m->head_swa % 16 || m->head_swa > 512 || m->n_rot_swa == 0 ||
m->n_rot_swa % 2 || m->n_rot_swa > m->head_swa || m->window == 0 ||
m->n_head % m->n_kv_swa || (m->n_head * m->head_swa) % JANAS_QK)
return -1;
return 0;
}
/*
* Rows of small weights read as f32 (a DeltaNet layer's alpha and beta):
* the file's own when they are f32, else made f32 once from whatever type
* they come in (Q8_0 or F16 in unsloth's files, Q5_1 when a whole file was
* requantized), exactly - d_model x value heads, too few to be worth a
* kernel.
*/
static int rows_f32(struct janas_llm_model *m, const struct janas_jns *j,
uint32_t l, const char *suffix, uint64_t d0, uint64_t d1,
const float **out, float **own, char *err, size_t err_len)
{
char name[64];
snprintf(name, sizeof(name), "blk.%u.%s", l, suffix);
const struct janas_jns_tensor *t = janas_jns_tensor(j, name);
int half = t && (t->type == 1 || t->type == 30); /* f16, bf16 */
if (!t || t->dims[0] != d0 || t->dims[1] != d1 ||
(t->type != 0 && !half &&
(janas_qtype_block_size((int)t->type) == 0 || d0 % JANAS_QK))) {
snprintf(err, err_len, "tensor %s missing or unexpected", name);
return -1;
}
if (t->type == 0) {
*out = f32(m, t);
return 0;
}
if (!(*own = malloc(d0 * d1 * sizeof(float)))) {
snprintf(err, err_len, "out of memory");
return -1;
}
if (half) {
const uint16_t *h = data(m, t);
for (uint64_t i = 0; i < d0 * d1; i++)
(*own)[i] = t->type == 1 ? janas_fp16_to_fp32(h[i])
: janas_u2f((uint32_t)h[i] << 16);
} else {
janas_dequantize((int)t->type, data(m, t), *own, d0 * d1);
}
*out = *own;
return 0;
}
/*
* head_bits 4: the output head, when the file holds it at six bits (Q6_K),
* requantized to Q4_K once, at load, before anything takes its type: the
* largest matrix read whole at every token, a seventh of Qwen3.5-9B's
* token. Into a buffer of its own, then copied where it was: the smaller
* copy fits there. Only the head: on Qwen3.5-9B, 400 tokens of Italian
* prose against llama.cpp, the most likely token agreed 375 times where
* 383 with the head requantized, 368 with every other six-bit matrix -
* attention, the shared experts - which gave next to no speed (7 Oct 2026).
* Where the head is the token embeddings (Qwen3-0.6B and 4B, Qwen3.5-0.8B
* and 2B) the table is requantized in place, so the embeddings read at four
* bits too, as in the larger models' files: Qwen3.5-2B +9%, the most
* likely token 352 where 376 (8 Oct 2026). A head at five bits (Q5_K,
* Qwen3.8-Flash-Next's from ISTA, 437 MB read at every token) is
* requantized the same way.
*/
struct rq_job {
const uint8_t *src;
uint8_t *dst;
size_t first, count; /* blocks of JANAS_QK weights */
int type; /* the source's: Q6_K or Q5_K */
};
static void *rq_worker(void *arg)
{
struct rq_job *r = arg;
float buf[JANAS_QK * 16];
size_t sb = janas_qtype_block_size(r->type),
db = janas_qtype_block_size(JANAS_Q4_K);
for (size_t b = r->first; b < r->first + r->count; b += 16) {
size_t nb = r->first + r->count - b < 16 ? r->first + r->count - b : 16;
janas_dequantize(r->type, r->src + b * sb, buf, nb * JANAS_QK);
janas_q4k_quantize_fast(
buf, (struct janas_block_q4k *)(r->dst + b * db), nb * JANAS_QK);
}
return NULL;
}
/*
* An output head or embedding table in BF16 or F16 - the MXFP4_MOE files
* keep output.weight in BF16, 1.02 GB for Qwen3.6-35B-A3B's 248,320 rows -
* rewritten once, at load, as Q8_0, as llama.cpp's quantize would make it:
* the products read quantized types, and the head, read whole at every
* token, is read at half the bytes (BF16 would cost some 12 ms a token at
* 81 GB/s). Built beside and copied in place, Q8_0 being the smaller.
*/
struct h16_job {
const uint8_t *src;
struct janas_block_q8_0 *dst;
size_t first, count; /* blocks of 32 weights */
int bf16;
};
static void *h16_worker(void *arg)
{
struct h16_job *r = arg;
float buf[32];
for (size_t b = r->first; b < r->first + r->count; b++) {
const uint16_t *s = (const uint16_t *)(r->src + b * 64);
for (int j = 0; j < 32; j++) {
if (r->bf16) {
uint32_t bits = (uint32_t)s[j] << 16;
memcpy(&buf[j], &bits, sizeof(float));
} else
buf[j] = janas_fp16_to_fp32(s[j]);
}
janas_q8_0_quantize(buf, r->dst + b, 32);
}
return NULL;
}
static void head16_to_q8_0(struct janas_llm_model *m, const char *name)
{
struct janas_jns_tensor *t =
(struct janas_jns_tensor *)janas_jns_tensor(&m->j, name);
if (!t || (t->type != 30 && t->type != 1) || t->dims[0] % JANAS_QK ||
t->offset < m->j.h.resident_offset ||
t->offset + t->bytes > m->j.h.resident_offset + m->j.h.resident_bytes)
return;
enum { NT = 16 };
long cpus = sysconf(_SC_NPROCESSORS_ONLN);
int nt = cpus < 1 ? 1 : cpus > NT ? NT : (int)cpus;
size_t nb = t->bytes / 64;
uint8_t *src = (uint8_t *)data(m, t);
struct janas_block_q8_0 *dst = malloc(nb * sizeof(*dst));
if (!dst)
return;
pthread_t th[NT];
struct h16_job jb[NT];
int started = 0;
for (int k = 0; k < nt; k++) {
size_t a = nb * k / nt, b = nb * (k + 1) / nt;
jb[k] = (struct h16_job){.src = src,
.dst = dst,
.first = a,
.count = b - a,
.bf16 = t->type == 30};
if (pthread_create(&th[k], NULL, h16_worker, &jb[k]) == 0)
started |= 1 << k;
else
h16_worker(&jb[k]);
}
for (int k = 0; k < nt; k++)
if (started & 1 << k)
pthread_join(th[k], NULL);
if (!strcmp(name, "output.weight") ||
!janas_jns_tensor(&m->j, "output.weight")) {
m->head16 = (int)t->type;
m->head16_before = t->bytes;
m->head16_after = nb * sizeof(*dst);
}
t->type = JANAS_Q8_0;
t->bytes = nb * sizeof(*dst);
memcpy(src, dst, t->bytes);
free(dst);
}
static void requant_head(struct janas_llm_model *m)
{
/* tied embeddings (the small Qwen): the table is the head */
struct janas_jns_tensor *t =
(struct janas_jns_tensor *)janas_jns_tensor(&m->j, "output.weight");
if (!t)
t = (struct janas_jns_tensor *)janas_jns_tensor(&m->j,
"token_embd.weight");
if (!t || (t->type != JANAS_Q6_K && t->type != JANAS_Q5_K) ||
t->dims[0] % JANAS_QK || t->offset < m->j.h.resident_offset ||
t->offset + t->bytes > m->j.h.resident_offset + m->j.h.resident_bytes)
return;
enum { NT = 16 };
long cpus = sysconf(_SC_NPROCESSORS_ONLN);
int nt = cpus < 1 ? 1 : cpus > NT ? NT : (int)cpus;
int type = (int)t->type;
size_t nb = t->bytes / janas_qtype_block_size(type);
uint8_t *src = (uint8_t *)data(m, t);
uint8_t *dst = malloc(nb * janas_qtype_block_size(JANAS_Q4_K));
if (!dst)
return;
pthread_t th[NT];
struct rq_job jb[NT];
int started = 0;
for (int k = 0; k < nt; k++) {
size_t a = nb * k / nt, b = nb * (k + 1) / nt;
jb[k] = (struct rq_job){
.src = src, .dst = dst, .first = a, .count = b - a, .type = type};
if (pthread_create(&th[k], NULL, rq_worker, &jb[k]) == 0)
started |= 1 << k;
else
rq_worker(&jb[k]);
}
for (int k = 0; k < nt; k++)
if (started & 1 << k)
pthread_join(th[k], NULL);
m->head_before = t->bytes;
t->type = JANAS_Q4_K;
t->bytes = nb * janas_qtype_block_size(JANAS_Q4_K);
memcpy(src, dst, t->bytes);
m->head_after = t->bytes;
free(dst);
}
static int load_layer(struct janas_llm_model *m, const struct janas_jns *j,
uint32_t l, struct layer *ly, char *err, size_t err_len)
{
if (!ly->hd) { /* the model's geometry, unless the caller set it */
ly->hd = m->head_dim;
ly->n_kv = m->n_head_kv;
ly->n_rot = m->n_rot;
}
uint32_t dm = m->d_model, qd = m->n_head * ly->hd, kvd = ly->n_kv * ly->hd,
ff = m->d_ff;
char name[64];
#define NEED(field, suffix, kind, d0, d1) \
snprintf(name, sizeof(name), "blk.%u." suffix, l); \
if (!(ly->field = need(j, name, kind, d0, d1, err, err_len))) \
return -1;
#define NEED_T(var, suffix, d0) \
snprintf(name, sizeof(name), "blk.%u." suffix, l); \
if (!(var = need(j, name, T_F32, d0, 0, err, err_len))) \
return -1;
/* qwen4exp: the hyper-connections' mixes in place of the norms */
if (!m->a->hc) {
NEED(attn_norm, "attn_norm.weight", T_F32, dm, 0)
}
if (ly->rec) {
uint32_t s = m->ds;
if (m->a->rec == JANAS_REC_Q35) {
NEED(ssm_qkv, "attn_qkv.weight", T_QUANT, dm, m->conv_ch)
NEED(ssm_z, "attn_gate.weight", T_QUANT, dm, m->d_inner)
if (rows_f32(m, j, l, "ssm_alpha.weight", dm, m->n_vh,
&ly->ssm_alpha, &ly->alpha_own, err, err_len) ||
rows_f32(m, j, l, "ssm_beta.weight", dm, m->n_vh, &ly->ssm_beta,
&ly->beta_own, err, err_len))
return -1;
} else {
/* qwen3next's fused projection, q, k, v and z interleaved per
key head; llama.cpp's converter writes it as two since 2026,
q|k|v and z each in the heads' order, as qwen35 has them
(issue #18: unsloth's Qwen3-Coder-Next files). ssm_ba is the
same in both. */
snprintf(name, sizeof(name), "blk.%u.attn_qkv.weight", l);
if (janas_jns_tensor(j, name)) {
NEED(ssm_qkv, "attn_qkv.weight", T_QUANT, dm, m->conv_ch)
NEED(ssm_z, "attn_gate.weight", T_QUANT, dm, m->d_inner)
} else {
NEED(ssm_in, "ssm_in.weight", T_QUANT, dm, m->qkvz_dim)
}
NEED(ssm_ba, "ssm_ba.weight", T_QUANT, dm, 2 * m->n_vh)
}
NEED(conv, "ssm_conv1d.weight", T_F32, m->d_conv, m->conv_ch)
ly->conv_t = malloc((size_t)m->d_conv * m->conv_ch * sizeof(float));
if (!ly->conv_t)
return -1;
for (uint32_t c = 0; c < m->conv_ch; c++)
for (uint32_t i = 0; i < m->d_conv; i++)
ly->conv_t[(size_t)i * m->conv_ch + c] =
f32(m, ly->conv)[(size_t)c * m->d_conv + i];
NEED(dt_bias, "ssm_dt.bias", T_F32, m->n_vh, 0)
NEED(ssm_a, "ssm_a", T_F32, m->n_vh, 0)
NEED(ssm_norm, "ssm_norm.weight", T_F32, s, 0)
NEED(ssm_out, "ssm_out.weight", T_QUANT, m->d_inner, dm)
} else {
NEED(wq, "attn_q.weight", T_QUANT, dm, qd * (m->a->attn_gate ? 2 : 1))
NEED(wo, "attn_output.weight", T_QUANT, qd, dm)
if (!m->a->no_qk_norm) {
NEED(q_norm, "attn_q_norm.weight", T_F32, ly->hd, 0)
}
/* a layer reading an earlier one's keys and values has no use for
its k and v, even where the file has them */
if (!m->n_kv_layers || l < m->n_kv_layers) {
NEED(wk, "attn_k.weight", T_QUANT, dm, kvd)
snprintf(name, sizeof(name), "blk.%u.attn_v.weight", l);
if (!m->a->swa || janas_jns_tensor(j, name)) {
NEED(wv, "attn_v.weight", T_QUANT, dm, kvd)
}
if (!m->a->no_qk_norm) {
NEED(k_norm, "attn_k_norm.weight", T_F32, ly->hd, 0)
}
}
}
if (m->ple_dim) {
const struct janas_jns_tensor *pt[2];
static const char *const pn[2] = {"inp_gate.weight", "proj.weight"};
for (int p = 0; p < 2; p++) {
snprintf(name, sizeof(name), "blk.%u.%s", l, pn[p]);
pt[p] = janas_jns_tensor(j, name);
uint64_t d0 = p ? m->ple_dim : dm, d1 = p ? dm : m->ple_dim;
if (!pt[p] || pt[p]->dims[0] != d0 || pt[p]->dims[1] != d1 ||
(pt[p]->type != 0 &&
janas_qtype_block_size((int)pt[p]->type) == 0)) {
snprintf(err, err_len, "tensor %s missing or unexpected", name);
return -1;
}
}
ly->ple_gate = pt[0];
ly->ple_proj = pt[1];
NEED(ple_post, "post_norm.weight", T_F32, dm, 0)
}
ly->out_scale = 1.0f;
if (m->a->sandwich) {
NEED(post_attn, "post_attention_norm.weight", T_F32, dm, 0)
NEED(post_ffn, "post_ffw_norm.weight", T_F32, dm, 0)
snprintf(name, sizeof(name), "blk.%u.layer_output_scale.weight", l);
const struct janas_jns_tensor *os = janas_jns_tensor(j, name);
if (os && os->type == 0 && os->dims[0] == 1)
ly->out_scale = f32(m, os)[0];
}
if (m->a->hc) {
if (janas_m_hc_layer_load(m, j, l, ly, err, err_len) != 0)
return -1;
} else if (m->a->post_attn_norm) {
NEED(ffn_norm, "post_attention_norm.weight", T_F32, dm, 0)
} else {
NEED(ffn_norm, "ffn_norm.weight", T_F32, dm, 0)
}
snprintf(name, sizeof(name), "blk.%u.ffn_gate_inp.weight", l);
ly->gate_inp = janas_jns_tensor(j, name);
if (!ly->gate_inp && m->n_expert == 1) {
/* a dense feed-forward is the only expert of its layer and has no
router: a zero logit comes out of the softmax as weight 1 exactly,
so the routed path below serves it unchanged */
ly->router_own = calloc(dm, sizeof(uint16_t));
if (!ly->router_own)
return -1;
ly->router16 = ly->router_own;
} else if (!ly->gate_inp || ly->gate_inp->dims[0] != dm ||
ly->gate_inp->dims[1] != m->n_expert ||
(ly->gate_inp->type != 0 && ly->gate_inp->type != 30)) {
snprintf(err, err_len, "tensor %s missing or unexpected", name);
return -1;
} else {
/* the router in half precision, converted in place: half the reads of
a table read in full for every token (from f32 or bf16) */
uint16_t *h16 = (uint16_t *)data(m, ly->gate_inp);
size_t nw = (size_t)dm * m->n_expert;
if (ly->gate_inp->type == 0) {
const float *w = f32(m, ly->gate_inp);
for (size_t i = 0; i < nw; i++)
h16[i] = janas_fp32_to_fp16(w[i]);
} else {
for (size_t i = 0; i < nw; i++)
h16[i] = janas_fp32_to_fp16(janas_u2f((uint32_t)h16[i] << 16));
}
ly->router16 = h16;
}
if (m->a->sandwich && ly->gate_inp) {
/* Gemma 4's mixture: the dense feed-forward beside the experts (as
wide as its file says, widened by gguf2jns), its norms, and the
router's and the experts' scales */
static const char *const dn[3] = {"ffn_gate.weight", "ffn_up.weight",
"ffn_down.weight"};
const struct janas_jns_tensor *sw[3];
for (int p = 0; p < 3; p++) {
snprintf(name, sizeof(name), "blk.%u.%s", l, dn[p]);
if (!(sw[p] = need(j, name, T_QUANT, p == 2 ? sw[0]->dims[1] : dm,
p == 2 ? dm : 0, err, err_len)))
return -1;
}
if (sw[0]->dims[1] != sw[1]->dims[1] || sw[0]->dims[1] % JANAS_QK ||
sw[2]->dims[0] != sw[0]->dims[1] || sw[2]->dims[1] != dm) {
snprintf(err, err_len, "layer %u: dense feed-forward shapes", l);
return -1;
}
for (int p = 0; p < 3; p++) {
ly->sh_w[p] = data(m, sw[p]);
ly->sh_mx[p] =
(struct janas_jns_matrix){.type = sw[p]->type,
.rows = (uint32_t)sw[p]->dims[1],
.cols = (uint32_t)sw[p]->dims[0],
.bytes = sw[p]->bytes};
}
NEED(pre_norm2, "pre_ffw_norm_2.weight", T_F32, dm, 0)
NEED(post_norm1, "post_ffw_norm_1.weight", T_F32, dm, 0)
NEED(post_norm2, "post_ffw_norm_2.weight", T_F32, dm, 0)
const struct janas_jns_tensor *rs, *es;
NEED_T(rs, "ffn_gate_inp.scale", dm)
NEED_T(es, "ffn_down_exps.scale", m->n_expert)
ly->router_scale = f32(m, rs);
ly->exp_scale = f32(m, es);
}
if (m->a->shared_expert) {
const struct janas_jns_tensor *sg, *sw[3];
static const char *const sh_names[3] = {"ffn_gate_shexp.weight",
"ffn_up_shexp.weight",
"ffn_down_shexp.weight"};
for (int p = 0; p < 3; p++) {
snprintf(name, sizeof(name), "blk.%u.%s", l, sh_names[p]);
if (!(sw[p] = need(j, name, T_QUANT, p == 2 ? ff : dm,
p == 2 ? dm : ff, err, err_len)))
return -1;
}
snprintf(name, sizeof(name), "blk.%u.ffn_gate_inp_shexp.weight", l);
sg = janas_jns_tensor(j, name);
if (!sg || sg->dims[0] != dm ||
(sg->type != 0 && sg->type != 1 && sg->type != 30)) {
snprintf(err, err_len, "tensor %s missing or unexpected", name);
return -1;
}
for (int p = 0; p < 3; p++) {
ly->sh_w[p] = data(m, sw[p]);
ly->sh_mx[p] =
(struct janas_jns_matrix){.type = sw[p]->type,
.rows = (uint32_t)sw[p]->dims[1],
.cols = (uint32_t)sw[p]->dims[0],
.bytes = sw[p]->bytes};
}
ly->sh_gate = malloc(dm * sizeof(float));
if (!ly->sh_gate)
return -1;
if (sg->type == 0) {
memcpy(ly->sh_gate, data(m, sg), dm * sizeof(float));
} else if (sg->type == 30) { /* bf16 */
const uint16_t *b16 = data(m, sg);
for (uint32_t i = 0; i < dm; i++)
ly->sh_gate[i] = janas_u2f((uint32_t)b16[i] << 16);
} else {
const uint16_t *h16 = data(m, sg);
for (uint32_t i = 0; i < dm; i++)
ly->sh_gate[i] = janas_fp16_to_fp32(h16[i]);
}
}
#undef NEED
#undef NEED_T
return 0;
}
/* The MTP block: a one-layer JNS file converted by tools/hf2jns_mtp. */
static int load_mtp(struct janas_llm_model *m, const char *path, char *err,
size_t err_len)
{
char why[256];
if (janas_jns_open(path, &m->mtp_j, why, sizeof(why)) != 0) {
/* the MTP file's own name: the caller names the model's */
snprintf(err, err_len, "MTP block %s: %s", path, why);
return -1;
}
m->mtp_open = 1;
const struct janas_jns_header *h = &m->mtp_j.h;
if (strncmp(h->arch, m->j.h.arch, sizeof(h->arch)) != 0 ||
h->n_layer != 1 || h->n_expert != m->n_expert ||
h->n_expert_used != m->k || h->d_model != m->d_model ||
m->mtp_j.layers[0].m[JANAS_JNS_GATE].rows != m->d_ff ||
m->mtp_j.layers[0].m[JANAS_JNS_DOWN].rows != m->d_model) {
snprintf(err, err_len, "%s: not an MTP block of this model", path);
return -1;
}
m->mtp_res = janas_aligned_alloc(4096, h->resident_bytes);
if (!m->mtp_res) {
snprintf(err, err_len, "out of memory for the MTP block");
return -1;
}
for (uint64_t done = 0; done < h->resident_bytes;) {
ssize_t r =
pread(m->mtp_j.fd, m->mtp_res + done, h->resident_bytes - done,
(off_t)(h->resident_offset + done));
if (r <= 0) {
snprintf(err, err_len, "cannot read the MTP block");
return -1;
}
done += (uint64_t)r;
}
uint32_t dm = m->d_model;
m->mtp = m->mtp_own_cache = 1;
m->mtp_jl = &m->mtp_j.layers[0];
m->mtp_cl = 0;
m->mtp_layer.rec = 0;
m->mtp_layer.slot = m->n_attn; /* the KV slot after the main model's */
if (load_layer(m, &m->mtp_j, 0, &m->mtp_layer, err, err_len) != 0 ||
!(m->eh_proj = need(&m->mtp_j, "mtp.eh_proj.weight", T_QUANT, 2 * dm,
dm, err, err_len)) ||
!(m->enorm =
need(&m->mtp_j, "mtp.enorm.weight", T_F32, dm, 0, err, err_len)))
return -1;
if (m->a->hc) {
/* qwen4exp's block (unsloth's GGUF heads, hf2jns_mtp): a norm for
each of the trunk's streams, and a final mix of its own in place
of a norm */
uint32_t hd = m->hc_n * dm;
if (!(m->hnorm = need(&m->mtp_j, "mtp.hnorm.weight", T_F32, hd, 0, err,
err_len)) ||
!(m->mtp_hco_norm = need(&m->mtp_j, "mtp.hc_head_norm.weight",
T_F32, hd, 0, err, err_len)))
return -1;
m->mtp_hco_down =
janas_jns_tensor(&m->mtp_j, "mtp.hc_head_down.weight");
m->mtp_hco_up = janas_jns_tensor(&m->mtp_j, "mtp.hc_head_up.weight");
if (!m->mtp_hco_down || !m->mtp_hco_up ||
m->mtp_hco_down->dims[0] != hd ||
m->mtp_hco_down->dims[1] != m->hc_rank ||
m->mtp_hco_up->dims[0] != m->hc_rank ||
m->mtp_hco_up->dims[1] != hd) {
snprintf(err, err_len, "%s: its final mix missing or unexpected",
path);
return -1;
}
return 0;
}
if (!(m->hnorm = need(&m->mtp_j, "mtp.hnorm.weight", T_F32, dm, 0, err,
err_len)) ||
!(m->mtp_norm =
need(&m->mtp_j, "mtp.norm.weight", T_F32, dm, 0, err, err_len)))
return -1;
return 0;
}
/* The MTP block inside the main file (qwen35moe: the block after the
model's layers, tensors nextn.*); its experts are in the main cache. */
static int load_mtp_internal(struct janas_llm_model *m, char *err,
size_t err_len)
{
uint32_t l = m->n_layer, dm = m->d_model;
char name[64];
m->mtp_layer.rec = 0;
m->mtp_layer.slot = m->n_attn;
if (load_layer(m, &m->j, l, &m->mtp_layer, err, err_len) != 0)
return -1;
#define NEXTN(field, suffix, kind, d0, d1) \
snprintf(name, sizeof(name), "blk.%u.nextn." suffix, l); \
if (!(m->field = need(&m->j, name, kind, d0, d1, err, err_len))) \
return -1;
NEXTN(eh_proj, "eh_proj.weight", T_QUANT, 2 * dm, dm)
NEXTN(enorm, "enorm.weight", T_F32, dm, 0)
NEXTN(hnorm, "hnorm.weight", T_F32, dm, 0)
NEXTN(mtp_norm, "shared_head_norm.weight", T_F32, dm, 0)
#undef NEXTN
m->mtp = 1;
m->mtp_own_cache = 0;
m->mtp_jl = &m->j.layers[l];
m->mtp_cl = l;
return 0;
}
/* The draft head with the base lowest ids, room for 8192 more tokens. */
static int init_draft_head(struct janas_llm_model *m, uint32_t base)
{
return janas_m_init_draft_head(
m, data(m, m->output),
m->d_model / JANAS_QK * janas_qtype_block_size((int)m->output->type),
base);
}
int janas_m_init_draft_head(struct janas_llm_model *m, const uint8_t *src,
size_t row_bytes, uint32_t base)
{
if (base > m->n_vocab)
base = m->n_vocab;
m->dh_src = src;
m->dh_row = row_bytes;
m->dh_cap = base + 8192 < m->n_vocab ? base + 8192 : m->n_vocab;
m->dh_w = malloc((size_t)m->dh_cap * m->dh_row);
m->dh_ids = malloc(m->dh_cap * sizeof(int32_t));
m->dh_in = calloc(m->n_vocab, 1);
m->dh_logits = malloc(m->dh_cap * sizeof(float));
if (!m->dh_w || !m->dh_ids || !m->dh_in || !m->dh_logits)
return -1;
memcpy(m->dh_w, src, (size_t)base * m->dh_row);
for (uint32_t i = 0; i < base; i++) {
m->dh_ids[i] = (int32_t)i;
m->dh_in[i] = 1;
}
m->dh_n = base;
return 0;
}
uint32_t janas_llm_main_layers(const struct janas_jns *j)
{
const struct janas_arch *a = janas_arch_find(j->h.arch, sizeof(j->h.arch));
char key[128];
uint32_t nextn = 0;
snprintf(key, sizeof(key), "%.32s.nextn_predict_layers", j->h.arch);
if (a && a->nextn_in_file && janas_jns_meta_u32(j, key, &nextn) == 0 &&
nextn < j->h.n_layer)
return j->h.n_layer - nextn; /* the file's last blocks predict ahead */
return j->h.n_layer;
}
/* A buffer a product writes: whole pages, so that the GPU can be given it
(janas_gpu_import), listed for that. */
static float *out_alloc(struct janas_llm_model *m, size_t bytes)
{
bytes = (bytes + 4095) & ~(size_t)4095;
float *p = janas_aligned_alloc(4096, bytes ? bytes : 4096);
if (p && m->n_out < (int)(sizeof(m->out) / sizeof(*m->out)))
m->out[m->n_out++] = (struct janas_out_buf){p, bytes};
return p;
}
float *janas_m_out_alloc(struct janas_llm_model *m, size_t bytes)
{
return out_alloc(m, bytes);
}
struct janas_llm_model *janas_llm_model_load(const char *path,
const struct janas_llm_options *o,
char *err, size_t err_len)
{
struct janas_llm_model *m = calloc(1, sizeof(*m));
if (!m)
return NULL;
if (janas_jns_open(path, &m->j, err, err_len) != 0) {
free(m);
return NULL;
}
m->a = janas_arch_find(m->j.h.arch, sizeof(m->j.h.arch));
if (!m->a) {
snprintf(err, err_len, "architecture %.32s not supported", m->j.h.arch);
goto fail;
}
if (!m->j.h.n_layer) { /* resident tensors only: a vision encoder */
snprintf(err, err_len, "%s holds no layers: not a language model",
path);
goto fail;
}
m->n_layer = janas_llm_main_layers(&m->j);
m->d_model = m->j.h.d_model;
m->n_expert = m->j.h.n_expert;
m->k = m->j.h.n_expert_used;
m->d_ff = m->j.layers[0].m[JANAS_JNS_GATE].rows;
uint32_t interval = 4;
const uint8_t *swa_pat = NULL;
if (m->a->nh
? janas_m_nh_params(m) != 0
: meta_u32(m, "attention.head_count", &m->n_head) ||
(m->a->swa ? load_swa_params(m, &swa_pat)
: meta_u32(m, "attention.head_count_kv",
&m->n_head_kv)) ||
meta_u32(m, "attention.key_length", &m->head_dim) ||
meta_f32(m, "attention.layer_norm_rms_epsilon", &m->eps) ||
meta_f32(m, "rope.freq_base", &m->rope_base) ||
m->k > MAX_K || m->n_head_kv == 0 ||
m->n_head % m->n_head_kv || m->head_dim % 8 ||
m->d_model % JANAS_QK ||
(m->n_head * m->head_dim) % JANAS_QK ||
(m->d_ff % JANAS_QK && !m->a->hc) ||
(m->a->rec && load_ssm_params(m) != 0)) {
snprintf(err, err_len, "missing or inconsistent hyperparameters");
goto fail;
}
if (meta_u32(m, "rope.dimension_count", &m->n_rot) != 0)
m->n_rot = m->head_dim;
/* the multimodal RoPE's sections, where the model has them */
{
char key[128];
uint64_t ns = 0;
const int32_t *sec = NULL;
snprintf(key, sizeof(key), "%.32s.rope.dimension_sections",
m->j.h.arch);
if (janas_jns_meta_ints(&m->j, key, &ns, &sec) == 0 && ns >= 3)
for (int i = 0; i < 3; i++) {
int32_t v;
memcpy(&v, sec + i, sizeof v);
m->mrope[i] = v > 0 ? (uint32_t)v : 0;
}
}
if (m->a->rec)
meta_u32(m, "full_attention_interval", &interval);
if (m->n_rot == 0 || m->n_rot % 2 || m->n_rot > m->head_dim ||
interval == 0 || m->head_dim > 512) {
snprintf(err, err_len, "inconsistent rotary or layer pattern");
goto fail;
}
if (m->a->hc && janas_m_hc_load(m, err, err_len) != 0)
goto fail;
m->hd_max = m->head_dim > m->head_swa ? m->head_dim : m->head_swa;
m->kvd_max = m->n_head_kv * m->head_dim;
if (m->n_kv_swa * m->head_swa > m->kvd_max)
m->kvd_max = m->n_kv_swa * m->head_swa;
/* the experts of a mixture are alike; a dense model's feed-forward
may widen from one layer to the next (Gemma 4's E2B: 6144, then
12288): the buffers take the widest */
for (uint32_t l = 0; l < m->n_layer && !m->a->nh; l++) {
uint32_t r = m->j.layers[l].m[JANAS_JNS_GATE].rows;
const struct janas_jns_matrix *gt = &m->j.layers[l].m[JANAS_JNS_GATE];
const struct janas_jns_matrix *up = &m->j.layers[l].m[JANAS_JNS_UP];
const struct janas_jns_matrix *dn = &m->j.layers[l].m[JANAS_JNS_DOWN];
/* by neuron (tools/jns_neurons.c), its rows are the gate's */
int by_neuron = dn->type == JANAS_Q6_K_PN;
/* a width that is no multiple of 256 (qwen4exp's 640): the down
matrix's rows end inside a group, which the types in blocks of
32 or 64 can do (janas_dot_rows_n) */
int narrow = r % JANAS_QK != 0;
/* every matrix the shape the buffers are made for: the file says
the shapes, and only the gate's rows were asked before */
if (gt->type == 0 || r == 0 || (r != m->d_ff && m->n_expert > 1) ||
gt->cols != m->d_model || up->rows != r || up->cols != m->d_model ||
(narrow &&
(!m->a->hc || janas_qtype_row_bytes((int)dn->type, r) == 0)) ||
(by_neuron ? dn->rows != r || dn->cols != m->d_model
: dn->rows != m->d_model || dn->cols != r)) {
snprintf(err, err_len, "layer %u: expert shapes differ", l);
goto fail;
}
if (r > m->d_ff)
m->d_ff = r;
}
/* Gemma 4's E2B and E4B: per-layer embeddings, and the last layers
reading the keys and values of earlier ones */
if (m->a->swa) {
uint32_t shared = 0;
meta_u32(m, "embedding_length_per_layer_input", &m->ple_dim);
meta_u32(m, "attention.shared_kv_layers", &shared);
if (shared && (shared + 2 > m->n_layer || !swa_pat)) {
snprintf(err, err_len, "inconsistent shared_kv_layers");
goto fail;
}
m->n_kv_layers = shared ? m->n_layer - shared : 0;
if (m->ple_dim &&
(m->ple_dim % JANAS_QK ||
!(m->ple_tok =
janas_jns_tensor(&m->j, "per_layer_token_embd.weight")) ||
m->ple_tok->dims[0] != (uint64_t)m->n_layer * m->ple_dim ||
janas_qtype_block_size((int)m->ple_tok->type) == 0 ||
m->ple_tok->type == JANAS_Q6_K_P ||
m->ple_tok->type == JANAS_Q6_K_PN ||
!(m->ple_model =
janas_jns_tensor(&m->j, "per_layer_model_proj.weight")) ||
m->ple_model->type != 30 || m->ple_model->dims[0] != m->d_model ||
m->ple_model->dims[1] != (uint64_t)m->n_layer * m->ple_dim ||
!(m->ple_norm =
janas_jns_tensor(&m->j, "per_layer_proj_norm.weight")) ||
m->ple_norm->type != 0 || m->ple_norm->dims[0] != m->ple_dim)) {
snprintf(err, err_len,
"per-layer embeddings missing or unexpected");
goto fail;
}
}
/* resident region into RAM; with the GPU, memory from its driver
(JANAS_GPU_ALLOC) or imported at the end */
const char *ge = getenv("JANAS_GPU");
snprintf(m->gpu_why, sizeof(m->gpu_why), "%s",
ge ? "turned off by JANAS_GPU=0" : "turned off (eco mode)");
if (ge ? atoi(ge) > 0 : o->use_gpu) {
/* the reason is kept: a tester with a GPU the engine does not take
has to be able to tell a build without GPU support from a driver
without Vulkan 1.3 */
char gerr[256] = "";
m->gpu = janas_gpu_create(gerr, sizeof(gerr));
snprintf(m->gpu_why, sizeof(m->gpu_why), "%.150s", m->gpu ? "" : gerr);
/* the resident region in the GPU driver's memory (JANAS_GPU_ALLOC,
or with the whole token on an integrated GPU): the GPU reads it
faster than imported pages, some 20% on the development laptop's */
if (m->gpu &&
(getenv("JANAS_GPU_ALLOC") ||
(getenv("JANAS_GPU_TOKEN") && janas_gpu_shares_host(m->gpu))) &&
(m->resident = janas_gpu_alloc(m->gpu, m->j.h.resident_bytes)))
m->resident_gpu = 1;
}
/* on 2 MB boundaries: see janas_m_huge_pages */
if (!m->resident)
m->resident = janas_aligned_alloc(
JANAS_HUGE_PAGE, (m->j.h.resident_bytes + JANAS_HUGE_PAGE - 1) /
JANAS_HUGE_PAGE * JANAS_HUGE_PAGE);
if (!m->resident) {
snprintf(err, err_len, "out of memory for the resident region");
goto fail;
}
/* all of it but the per-layer embeddings' table, read a row at a time
where it is (model_ple.c, model_hc.c): its pages, never touched, take
no memory - unless the machine holds it (ple_ram, JANAS_PLE_RAM):
then it is read with the rest, and its rows come from memory */
const char *pre = getenv("JANAS_PLE_RAM");
m->ple_ram = m->ple_tok && (pre ? atoi(pre) > 0 : o->ple_ram);
uint64_t skip_a = m->j.h.resident_bytes, skip_b = skip_a;
if (m->ple_tok && m->ple_tok->offset >= m->j.h.resident_offset) {
skip_a = (m->ple_tok->offset - m->j.h.resident_offset) & ~4095ull;
skip_b =
(m->ple_tok->offset - m->j.h.resident_offset + m->ple_tok->bytes) &
~4095ull;
if (skip_b < skip_a)
skip_b = skip_a;
m->res_hole[0] = skip_a, m->res_hole[1] = skip_b;
}
for (uint64_t done = 0; done < m->j.h.resident_bytes;) {
if (done == skip_a && skip_b > skip_a && !m->ple_ram) {
done = skip_b;
continue;
}
uint64_t end =
done < skip_a && !m->ple_ram ? skip_a : m->j.h.resident_bytes;
ssize_t r = pread(m->j.fd, m->resident + done, end - done,
(off_t)(m->j.h.resident_offset + done));
if (r <= 0) {
snprintf(err, err_len, "cannot read the resident region");
goto fail;
}
done += (uint64_t)r;
}
head16_to_q8_0(m, "output.weight");
head16_to_q8_0(m, "token_embd.weight");
const char *hbe = getenv("JANAS_HEAD_BITS");
if ((hbe ? atoi(hbe) : o->head_bits) == 4 && !m->resident_gpu)
requant_head(m);
if (m->a->hc && janas_m_hc_requant(m, err, err_len) != 0)
goto fail;
uint32_t dm = m->d_model, qd = m->n_head * m->hd_max, kvd = m->kvd_max,
ff = m->d_ff;
if (!(m->tok_embd =
need(&m->j, "token_embd.weight", T_QUANT, dm, 0, err, err_len)) ||
(!m->a->hc && !(m->out_norm = need(&m->j, "output_norm.weight", T_F32,
dm, 0, err, err_len))))
goto fail;
/* tied embeddings (the dense Qwen3): the table is also the output head */
m->output = janas_jns_tensor(&m->j, "output.weight")
? need(&m->j, "output.weight", T_QUANT, dm, 0, err, err_len)
: m->tok_embd;
if (!m->output)
goto fail;
m->n_vocab = (uint32_t)m->output->dims[1];
if (m->tok_embd->dims[1] != m->n_vocab) {
snprintf(err, err_len, "embedding and output vocabularies differ");
goto fail;
}
m->layers = calloc(m->n_layer, sizeof(*m->layers));
if (!m->layers)
goto fail;
for (uint32_t l = 0; l < m->n_layer; l++) {
struct layer *ly = &m->layers[l];
ly->rec = m->a->rec && (l + 1) % interval != 0;
if (m->a->nh) {
/* one part a layer: only the Mamba-2 ones keep a state, only
the attentions keys and values */
ly->nh_kind = janas_m_nh_kind(m, l);
ly->rec = ly->nh_kind == JANAS_NH_MAMBA;
ly->hd = m->head_dim;
ly->n_kv = (uint32_t)m->n_head_kv;
ly->n_rot = 0; /* no RoPE */
ly->slot = ly->rec ? m->n_rec++
: ly->nh_kind == JANAS_NH_ATTN ? m->n_attn++
: UINT32_MAX;
if (janas_m_nh_layer_load(m, l, ly, err, err_len) != 0)
goto fail;
continue;
}
if (swa_pat) {
ly->swa = swa_pat[l] != 0;
ly->hd = ly->swa ? m->head_swa : m->head_dim;
ly->n_kv = ly->swa ? m->n_kv_swa : m->n_head_kv;
ly->n_rot = ly->swa ? m->n_rot_swa : m->n_rot;
}
if (m->n_kv_layers && l >= m->n_kv_layers) {
/* the slot of the last layer of its kind that has one, as
llama.cpp takes it: the one before last, or the last */
uint32_t src = m->n_kv_layers - (ly->swa ? 2 : 1);
if (m->layers[src].swa != ly->swa) {
snprintf(err, err_len, "layer %u: no keys and values to share",
l);
goto fail;
}
ly->slot = m->layers[src].slot;
} else {
ly->slot = ly->rec ? m->n_rec++ : m->n_attn++;
}
if (load_layer(m, &m->j, l, ly, err, err_len) != 0)
goto fail;
}
/* the length the model was trained for: asking for more without a rope
scaling the file does not carry gives text that falls apart, so the
context stops there and the caller is told (janas_llm_describe) */
m->n_ctx_train = 0;
meta_u32(m, "context_length", &m->n_ctx_train);
m->n_ctx = o->n_ctx;
if (m->n_ctx_train && m->n_ctx > m->n_ctx_train)
m->n_ctx = m->n_ctx_train;
/* Gemma 4's assistant is opened last, over the caches it reads */
int assist = o->mtp_path && janas_m_assist_is(o->mtp_path);
/* the draft head's lowest token ids: the caller's, JANAS_DRAFT_VOCAB
(to experiment), or 32768 */
uint32_t dvocab = o->draft_vocab ? o->draft_vocab : 32768;
if (getenv("JANAS_DRAFT_VOCAB") && atol(getenv("JANAS_DRAFT_VOCAB")) > 0)
dvocab = (uint32_t)atol(getenv("JANAS_DRAFT_VOCAB"));
if (o->mtp_path && !assist && load_mtp(m, o->mtp_path, err, err_len) != 0)
goto fail;
if (!m->mtp && m->j.h.n_layer > m->n_layer &&
load_mtp_internal(m, err, err_len) != 0)
goto fail;
{
/* the full layers' RoPE factors (theta divided by them, pair by
pair: Gemma 4 turns only the first quarter of a head; Llama 3's
long context) */
const struct janas_jns_tensor *rf =
janas_jns_tensor(&m->j, "rope_freqs.weight");
if (rf && (rf->type != 0 || rf->dims[0] != m->n_rot / 2)) {
snprintf(err, err_len, "tensor rope_freqs.weight: unexpected");
goto fail;
}
m->rope_freqs = rf ? f32(m, rf) : NULL;
}
/* the KV cache, slot after slot, each with its own geometry, at eight
bits a number or sixteen */
{
const char *kv = getenv("JANAS_KV");
m->kv16 = kv ? atoi(kv) == 16 : m->a->kv16;
}
m->blk = janas_llm_prefill_block(&m->j, o ? o->min_block : 0);
/* the sliding-window layers keep a ring of positions, not the whole
context: the window and a prefill block (JANAS_SWA_RING=0 keeps
every position, as the full layers do) */
uint32_t ring_cap = 0;
if (m->a->swa) {
const char *rg = getenv("JANAS_SWA_RING");
ring_cap = janas_attn_ring_positions(m->window, m->blk);
if ((rg && atoi(rg) == 0) || ring_cap >= m->n_ctx)
ring_cap = 0;
}
uint32_t n_slots = m->n_attn + (m->mtp ? 1 : 0);
m->kv_cap = calloc(n_slots + 1, sizeof(uint32_t));
m->kv_ring = calloc(n_slots + 1, 1);
m->kv_swa = calloc(n_slots + 1, 1);
m->kv_off = malloc((n_slots + 1) * sizeof(size_t));
m->ks_off = malloc((n_slots + 1) * sizeof(size_t));
m->kv_hd = calloc(n_slots + 1, sizeof(uint32_t));
m->kv_heads = calloc(n_slots + 1, sizeof(uint32_t));
if (!m->kv_off || !m->ks_off || !m->kv_hd || !m->kv_heads || !m->kv_cap ||
!m->kv_ring || !m->kv_swa) {
snprintf(err, err_len, "out of memory");
goto fail;
}
for (uint32_t l = 0; l <= m->n_layer; l++) {
const struct layer *ly = l < m->n_layer ? &m->layers[l] : &m->mtp_layer;
if (ly->rec || (l == m->n_layer && !m->mtp) ||
(m->a->nh && l < m->n_layer && ly->nh_kind == JANAS_NH_MOE))
continue;
m->kv_hd[ly->slot] = ly->hd;
m->kv_heads[ly->slot] = ly->n_kv;
m->kv_swa[ly->slot] = (uint8_t)ly->swa;
m->kv_ring[ly->slot] = ly->swa && ring_cap;
m->kv_cap[ly->slot] = m->kv_ring[ly->slot] ? ring_cap : m->n_ctx;
}
size_t kv_len = 0, ks_len = 0;
for (uint32_t s = 0; s < n_slots; s++) {
m->kv_off[s] = kv_len;
m->ks_off[s] = ks_len;
kv_len += ((size_t)m->kv_cap[s] * m->kv_heads[s] * m->kv_hd[s])
<< m->kv16;
ks_len += (size_t)m->kv_cap[s] * m->kv_heads[s];
}
m->kv_off[n_slots] = kv_len;
m->ks_off[n_slots] = ks_len;
/* whole pages: an integrated GPU imports them for the token it runs
(gpu_token.h) */
size_t kvb = (kv_len + 4095) / 4096 * 4096,
ksb = (ks_len * sizeof(float) + 4095) / 4096 * 4096;
m->kcache = janas_aligned_alloc(4096, kvb ? kvb : 4096);
m->vcache = janas_aligned_alloc(4096, kvb ? kvb : 4096);
m->kscale = janas_aligned_alloc(4096, ksb ? ksb : 4096);
m->vscale = janas_aligned_alloc(4096, ksb ? ksb : 4096);
m->kv_bytes = kvb;
m->ks_bytes = ksb;
/* a pair's rows: the experts', or a wider shared feed-forward's */
m->ff_buf = m->d_ff;
for (uint32_t l = 0; l < m->n_layer; l++)
if (m->layers[l].sh_w[0] && m->layers[l].sh_mx[0].rows > m->ff_buf)
m->ff_buf = m->layers[l].sh_mx[0].rows;
/* in whole groups of 256: a narrower width's activations are quantized
with zeros after them (model_ffn.c) */
m->ff_buf = (m->ff_buf + JANAS_QK - 1) / JANAS_QK * JANAS_QK;
ff = m->ff_buf;
size_t B = m->blk, P = B * (m->k + 1);
size_t inner = m->d_inner > qd ? m->d_inner : qd;
size_t bigw = m->qkvz_dim > 2 * qd ? m->qkvz_dim : 2 * qd;
m->x = out_alloc(m, B * dm * sizeof(float)); /* the GPU reads it */
m->xn = out_alloc(m, B * dm * sizeof(float));
m->q = out_alloc(m, B * qd * sizeof(float));
m->kk = out_alloc(m, B * kvd * sizeof(float));
m->vv = out_alloc(m, B * kvd * sizeof(float));
m->att = malloc(B * inner * sizeof(float));
m->router = malloc(B * m->n_expert * sizeof(float));
m->rope_cos = malloc(B * m->n_rot / 2 * sizeof(float));
m->rope_sin = malloc(B * m->n_rot / 2 * sizeof(float));
if (m->a->swa) {
m->rope_cos_swa = malloc(B * m->n_rot_swa / 2 * sizeof(float));
m->rope_sin_swa = malloc(B * m->n_rot_swa / 2 * sizeof(float));
m->xres = malloc(B * dm * sizeof(float));
m->xbr = malloc(B * dm * sizeof(float));
if (!m->rope_cos_swa || !m->rope_sin_swa || !m->xres || !m->xbr) {
snprintf(err, err_len, "out of memory");
goto fail;
}
}
if (m->ple_dim) {
size_t W = (size_t)m->n_layer * m->ple_dim;
/* where the GPU writes the branches' products (janas_gpu_dense_mm) */
m->ple = out_alloc(m, B * W * sizeof(float));
m->ple_g = out_alloc(m, B * m->ple_dim * sizeof(float));
m->ple_row = malloc(2 * W * sizeof(float));
m->ple_q = malloc(B * m->ple_dim / JANAS_QK * sizeof(*m->ple_q));
if (!m->ple || !m->ple_g || !m->ple_row || !m->ple_q) {
snprintf(err, err_len, "out of memory");
goto fail;
}
}
m->xq = malloc(B * dm / JANAS_QK * sizeof(*m->xq));
m->attq = malloc(B * inner / JANAS_QK * sizeof(*m->attq));
m->pair_tok = malloc(P * sizeof(uint32_t));
m->pair_exp = malloc(P * sizeof(uint32_t));
m->order = malloc(P * sizeof(uint32_t));
m->glist = malloc(P * sizeof(uint32_t));
m->count = malloc((m->n_expert + 1) * sizeof(uint32_t));
m->pair_w = malloc(P * sizeof(float));
m->gate = out_alloc(m, P * ff * sizeof(float));
m->up = out_alloc(m, P * ff * sizeof(float));
m->h = malloc(P * ff * sizeof(float));
if (janas_m_ana_init(m, P) != 0) {
snprintf(err, err_len, "out of memory for the Ana-Net probe");
goto fail;
}
if (janas_m_janet_init(m) != 0) {
snprintf(err, err_len, "the Janet probe could not start");
goto fail;
}
if (janas_m_neurons_init(m, P) != 0) {
snprintf(err, err_len, "out of memory for the neurons' levels");
goto fail;
}
m->dout = out_alloc(m, P * dm * sizeof(float));
m->xg = malloc(P * dm / JANAS_QK * sizeof(*m->xg));
m->hq = malloc(P * ff / JANAS_QK * sizeof(*m->hq));
m->hlen = malloc(P * sizeof(uint32_t));
m->tok_first = malloc((B + 1) * sizeof(uint32_t));
m->tok_fill = malloc(B * sizeof(uint32_t));
m->tok_list = malloc(P * sizeof(uint32_t));
if (!m->tok_first || !m->tok_fill || !m->tok_list || !m->kcache ||
!m->vcache || !m->kscale || !m->vscale || !m->x || !m->xn || !m->q ||
!m->kk || !m->vv || !m->att || !m->router || !m->rope_cos ||
!m->rope_sin || !m->xq || !m->attq || !m->pair_tok || !m->pair_exp ||
!m->order || !m->glist || !m->count || !m->pair_w || !m->gate ||
!m->up || !m->h || !m->dout || !m->xg || !m->hq || !m->hlen ||
m->n_expert > 4096) {
snprintf(err, err_len, "out of memory");
goto fail;
}
#if defined(__x86_64__)
m->avx2 = __builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma");
#endif
m->trace = getenv("JANAS_TRACE") ? atoi(getenv("JANAS_TRACE")) : 0;
/* how much of an expert slot to read: the file may hold the down matrix
in bit planes, and a machine short of memory reads fewer of them */
{
int bits = o->expert_bits;
const char *eb = getenv("JANAS_EXPERT_BITS");
if (eb)
bits = atoi(eb);
for (uint32_t l = 0; l < m->j.h.n_layer; l++)
m->exp_planes |= m->j.layers[l].plane[0].type != 0;
if (!bits && m->exp_planes) {
/* nobody asked: keep every bit when the experts all fit in
memory, since then nothing is read while generating anyway;
otherwise four bits, which read a third to a half fewer bytes
and cost five to seven positions in four hundred */
bits = o->cache_bytes >= m->j.h.experts_bytes ||
o->min_quality >= JANAS_LLM_QUALITY_HIGH
? 6
: 4;
m->exp_auto = 1;
}
m->exp_level = bits == 2 ? 1 : bits == 4 ? 2 : 3;
if (!m->exp_planes) /* a file without planes: every level is the whole
slot, and the weights are the quantized ones */
m->exp_level = 3;
}
m->k_use = m->k;
/*
* Where the cache holds too small a share of the experts and the machine
* would wait for the disk at every token, the light experts that are not
* in RAM are left out (skip_misses): below 0.95/k of a token's weight
* with a cache of less than a tenth of the experts, below 0.8/k with
* less than a fifth. Until 8 Oct 2026 the lightest experts were left out
* whether in RAM or not - six of ten, or eight - which gave up more for
* less. Qwen3-Next-80B, four bits, 400 tokens against llama.cpp (most
* likely token agreeing, mean |logit difference|, tokens/s):
* C, 2 GiB all ten 393 0.17 11.0, six 383 0.53 20.9, skip 380
* 0.44 26.4 C, 4 GiB all ten 393 0.17 20.7, six 383 0.53 27.6, skip 384
* 0.36 27.7 C, 8 GiB all ten 393 0.17 26.0, eight 385 0.32 28.1, skip
* 391 0.25 27.1 Italian, 4 all ten 382 0.14 13.7, six 333 0.46 18.9,
* (0.8/k) 359 0.23 19.4 Italian, 8 all ten 382 0.14 18.7, eight 361
* 0.26 21.4, skip 372 0.23 23.7
*/
if (o->cache_bytes && m->j.h.experts_bytes) {
double cover = (double)o->cache_bytes / (double)m->j.h.experts_bytes;
/* below MEDIUM only: the caller's floor (min_quality) */
if (cover < 0.20 && o->min_quality < JANAS_LLM_QUALITY_MEDIUM) {
m->miss_skip = (cover < 0.10 ? 0.95f : 0.8f) / (float)m->k;
m->k_auto = 1;
}
}
if (getenv("JANAS_EXPERTS")) /* experts per token, for measures */
janas_llm_model_set_experts(m, (uint32_t)atoi(getenv("JANAS_EXPERTS")));
{
/* experts not in RAM left out below this weight, for measures (0:
never); at least one expert a token always stays, since the
heaviest weighs 1/k or more */
const char *ms = getenv("JANAS_MISS_SKIP");
if (ms)
m->miss_skip = (float)atof(ms);
if (m->miss_skip >= 1.0f / (float)m->k)
m->miss_skip = 0.0f;
}
{
const char *vm = getenv("JANAS_VMAP");
m->vmap_mod = vm ? strcmp(vm, "mod") == 0 : m->a->vmap_mod;
}
#if defined(__x86_64__)
m->f16c = __builtin_cpu_supports("f16c");
#endif
if (m->a->attn_gate || m->a->rec) {
/* the gated queries, and the DeltaNet's fused projections */
m->big = out_alloc(m, B * bigw * sizeof(float));
m->qgate = malloc(B * qd * sizeof(float));
if (!m->big || !m->qgate) {
snprintf(err, err_len, "out of memory");
goto fail;
}
}
/* the experts' input and output: the model's width, the block, x
(Nemotron-H's mixtures switch to their latent ones for a call) */
m->moe_w = dm;
m->moe_xq = m->xq;
m->moe_out = m->x;
if (m->a->nh && janas_m_nh_alloc(m) != 0) {
snprintf(err, err_len, "out of memory");
goto fail;
}
if (m->a->hc && (janas_m_hc_alloc(m, err, err_len) != 0 ||
janas_m_qsa_load(m, err, err_len) != 0))
goto fail;
if (m->a->rec) {
/* the GPU reads and writes them too (gpu_token.c): imported */
size_t sb = 2 * (size_t)m->n_rec * m->ssm_per * sizeof(uint16_t),
cb = 2 * (size_t)m->n_rec * m->conv_ch * (m->d_conv - 1) *
sizeof(float);
m->ssm_buf = (uint16_t *)out_alloc(m, sb);
m->conv_buf = out_alloc(m, cb);
if (m->ssm_buf)
memset(m->ssm_buf, 0, sb);
if (m->conv_buf)
memset(m->conv_buf, 0, cb);
m->ba = malloc(B * 2 * m->n_vh * sizeof(float));
/* the rows of the replayed tokens (a logged call, at most
JANAS_LLM_MAX_BLOCK), then the block */
size_t L = JANAS_LLM_MAX_BLOCK, R = L + B;
m->mixed = malloc(R * m->conv_ch * sizeof(float));
m->conv_out = malloc(R * m->conv_ch * sizeof(float));
m->beta = malloc(R * m->n_vh * sizeof(float));
m->decay = malloc(R * m->n_vh * sizeof(float));
/* the log too, which the GPU writes for a chain of drafts it
verifies and reads back to replay the tokens kept */
m->log_mixed =
out_alloc(m, (size_t)m->n_rec * L * m->conv_ch * sizeof(float));
m->log_beta =
out_alloc(m, (size_t)m->n_rec * L * m->n_vh * sizeof(float));
m->log_decay =
out_alloc(m, (size_t)m->n_rec * L * m->n_vh * sizeof(float));
if (!m->ssm_buf || !m->conv_buf || !m->ba || !m->mixed ||
!m->conv_out || !m->beta || !m->decay || !m->log_mixed ||
!m->log_beta || !m->log_decay) {
snprintf(err, err_len, "out of memory");
goto fail;
}
}
struct janas_cpu_layout lay;
int pin = -1;
if (o->compute_cpus &&
janas_cpu_layout(o->compute_cpus, o->n_compute, &lay) == 0) {
m->compute = o->shared_compute ? o->shared_compute
: janas_pool_create(lay.n, lay.cpus);
m->own_compute = !o->shared_compute;
m->p_cores = lay.n_p_cores;
m->p_threads = lay.n_p_threads;
m->all_threads = lay.n;
pin = o->shared_compute ? -1 : lay.cpus[0];
} else {
m->compute = o->shared_compute ? o->shared_compute
: janas_pool_create(o->n_compute, NULL);
m->own_compute = !o->shared_compute;
m->p_cores = m->p_threads = m->all_threads = o->n_compute;
}
m->io = janas_pool_create(o->n_io, o->io_cpus);
if (!m->compute || !m->io)
goto fail;
/* the I/O threads wait for NVMe reads of ~100 us: sleeping costs them
nothing and leaves the power budget to the compute cores */
janas_pool_set_spin(m->io, 0);
m->attn = janas_attn_create(m->n_head, m->n_head_kv, m->head_dim, m->n_ctx,
janas_pool_size(m->compute), o->attn_scores);
if (m->attn)
janas_attn_set_kv16(m->attn, m->kv16);
if (m->attn && m->a->swa) {
/* Gemma 4: the q and k norms scale the scores; the sliding-window
layers have an attention of their own */
janas_attn_set_scale(m->attn, 1.0f);
m->attn_swa = janas_attn_create(
m->n_head, m->n_kv_swa, m->head_swa, ring_cap ? ring_cap : m->n_ctx,
janas_pool_size(m->compute), o->attn_scores);
if (m->attn_swa) {
janas_attn_set_ring(m->attn_swa, ring_cap != 0);
janas_attn_set_kv16(m->attn_swa, m->kv16);
janas_attn_set_scale(m->attn_swa, 1.0f);
janas_attn_set_window(m->attn_swa, m->window);
}
}
if (!m->attn || (m->a->swa && !m->attn_swa)) {
snprintf(err, err_len, "out of memory");
goto fail;
}
if (assist && (janas_m_assist_load(m, o->mtp_path, o->attn_scores, err,
err_len) != 0 ||
janas_m_assist_head(m, dvocab) != 0))
goto fail;
/* the caller pinned for the time the model lives, and given its CPUs
back when it goes: a model opened after another in the same process
(the chat measuring, a reload) would otherwise see one CPU */
_Static_assert(JANAS_CPU_SAVE_BYTES <= sizeof(m->caller_cpus), "cpu set");
if (pin >= 0) {
m->caller_pinned = janas_cpus_save(m->caller_cpus) == 0;
janas_pin_current_thread(pin);
}
/* an MTP block in the file has its experts in this cache, in slots of
their own where they are larger (issue #34); one in a file of its own
has a cache of its own, and the file's block is never asked for */
m->cache = janas_expert_cache_create_split(
&m->j, m->mtp_own_cache ? m->n_layer : m->j.h.n_layer, m->n_layer,
o->cache_bytes, m->exp_level, m->io, err, err_len);
if (!m->cache)
goto fail;
/* reads ahead (model_ffn.c janas_m_prefetch): where they pay, of the
next layer's first 10 experts at most 2 (measured on 30 Sep 2026);
JANAS_PREFETCH=0 never, =r,b always with those */
if (janas_expert_cache_can_prefetch(m->cache)) {
const char *pf = getenv("JANAS_PREFETCH");
unsigned pr = 0, pb = 0;
m->pf_rank = 10;
m->pf_max = 2;
m->pf_mode = 1;
if (pf && atoi(pf) == 0) {
m->pf_mode = 0;
} else if (pf && sscanf(pf, "%u,%u", &pr, &pb) == 2 && pr >= 1 &&
pr <= 32 && pb >= 1) {
m->pf_rank = pr;
m->pf_max = pb;
m->pf_mode = 2;
m->pf_on = 1;
}
/* and in prefill blocks, where they pay (model.c): Qwen3-Next-80B
with a 4 GiB cache read 4096 tokens at 41.0 token/s against 34.6
without (6 Oct 2026); JANAS_PREFETCH_PREFILL=0 never, =2 always */
const char *pfb = getenv("JANAS_PREFETCH_PREFILL");
m->pfb_mode = 1;
if (pfb && strcmp(pfb, "0") == 0)
m->pfb_mode = 0;
else if (pfb && strcmp(pfb, "2") == 0)
m->pfb_mode = m->pfb_on = 2;
}
/* fill the cache with the experts this machine used most, before the
first reply has to find them one token at a time. A caller that does
not want this (a measurement) neither reads the profile nor adds to
it, so its runs leave the machine's profile as it was */
{
const char *w = getenv("JANAS_WARM");
size_t used = 0;
m->warm_use = w ? atoi(w) != 0 : o->warm != 0;
uint32_t *counts = m->warm_use ? janas_m_profile_read(m, &used) : NULL;
if (m->warm_use) {
m->use_count = calloc((size_t)m->j.h.n_layer * m->j.h.n_expert,
sizeof(uint32_t));
if (!m->use_count) {
free(counts);
goto fail;
}
}
if (counts) {
/* on a thread of its own: the first reply does not wait for it */
if (janas_expert_cache_warm_background(m->cache, counts) == 0) {
size_t slots = janas_expert_cache_slots(m->cache);
m->warm_experts = used < slots ? used : slots;
}
free(counts);
}
}
if (m->mtp) {
/* qwen4exp: the residual's streams a row, and the block's input a
stream at a time (model_hc.c janas_m_hc_mtp) */
size_t hw = m->hc_n ? m->hc_n : 1;
m->hid = malloc(B * hw * dm * sizeof(float));
m->mtp_hid = malloc(B * hw * dm * sizeof(float));
m->cat = malloc(B * 2 * dm * sizeof(float));
m->catq = malloc(B * hw * 2 * dm / JANAS_QK * sizeof(*m->catq));
/* its own experts' cache, or the main one (block in the file). Its
own holds every expert unless the caller says otherwise - the
API counts the whole file in the memory it plans (api.c
mem_fixed) - and never fewer than a block of tokens asks for at
once (model_ffn.c): 256 MiB, the old default, held 74 of
Qwen3.8-Flash-Next's experts of 3.6 MB, and a block asks for up
to JANAS_EXPERT_CHUNK */
uint64_t mtp_budget = 0;
if (m->mtp_own_cache) {
uint32_t ne = (uint32_t)m->mtp_j.h.n_expert;
uint64_t all =
janas_expert_cache_room(&m->mtp_j, 1, 1, m->exp_level);
uint64_t least =
all / ne * (ne < JANAS_EXPERT_CHUNK ? ne : JANAS_EXPERT_CHUNK);
mtp_budget = o->mtp_cache_bytes ? o->mtp_cache_bytes : all;
if (mtp_budget < least)
mtp_budget = least;
}
m->mtp_cache =
m->mtp_own_cache
? janas_expert_cache_create(&m->mtp_j, mtp_budget, m->exp_level,
m->io, err, err_len)
: m->cache;
if (!m->hid || !m->mtp_hid || !m->cat || !m->catq || !m->mtp_cache ||
init_draft_head(m, dvocab) != 0)
goto fail;
}
if (m->gpu) {
/* optional: without it (or if the weights cannot be shared with
it) everything runs on the CPU, with the same results. Memory
of its driver written by the CPU (reads, converted router) is
flushed out of the CPU's caches before the GPU reads it */
if (m->resident_gpu)
janas_gpu_flush(m->gpu, m->resident, m->j.h.resident_bytes);
else if (janas_m_huge_pages(m->resident, m->res_hole[1] > 0
? m->res_hole[0]
: m->j.h.resident_bytes),
janas_m_huge_pages(m->resident + m->res_hole[1],
m->res_hole[1] > 0
? m->j.h.resident_bytes - m->res_hole[1]
: 0),
janas_m_gpu_import(m, m->resident, m->j.h.resident_bytes, 0) !=
0)
m->gpu_off = 1;
if (m->mtp_res && janas_gpu_import(m->gpu, m->mtp_res,
m->mtp_j.h.resident_bytes) != 0)
m->gpu_off = 1;
/* the products' outputs too, so that the GPU writes them where the
engine reads them: through a staging buffer they cost a copy on
one thread, 0.4-2.8 ms a block of a layer on Qwen3-4B (9 Oct
2026). One that cannot be shared keeps the copy. Only where the
GPU shares the host's memory: on a card with memory of its own an
import is a copy there, which the GPU's writes never left - the
CPU read stale outputs, a Quadro RTX 5000 garbled its answers
(issue #17) */
if (janas_gpu_shares_host(m->gpu))
for (int i = 0; i < m->n_out; i++)
janas_gpu_import(m->gpu, m->out[i].p, m->out[i].bytes);
}
if (o->no_npu)
snprintf(m->npu_why, sizeof m->npu_why,
"left off: the expert cache needs the memory its copies "
"would take (--prompt-on npu asks for it)");
else
janas_m_npu_open(m);
janas_m_init_tuner(m);
return m;
fail:
janas_llm_model_free(m);
return NULL;
}
void janas_llm_model_free(struct janas_llm_model *m)
{
if (m)
janas_m_media_free(m);
if (janas_m_exp_tracing())
janas_m_exp_report();
if (m && getenv("JANAS_GPU_VERBOSE") &&
(m->gtok_runs[0] | m->gtok_runs[1] | m->gtok_runs[2]))
fprintf(stderr,
"janas: whole on the GPU %llu single tokens, %llu passes of "
"2-16, %llu blocks; the state %s (%llu there, %llu back)\n",
(unsigned long long)m->gtok_runs[0],
(unsigned long long)m->gtok_runs[1],
(unsigned long long)m->gtok_runs[2],
m->gtok_dev ? "copied to the card's memory" : "shared",
(unsigned long long)m->gtok_push,
(unsigned long long)m->gtok_pull);
if (m && m->skip_asked && (getenv("JANAS_MISS_SKIP") || m->trace))
fprintf(stderr,
"janas: experts not in RAM below %.3f left out: %llu of %llu "
"routed (%.1f%%)\n",
m->miss_skip,
(unsigned long long)(m->skip_asked - m->skip_left),
(unsigned long long)m->skip_asked,
100.0 * (double)(m->skip_asked - m->skip_left) /
(double)m->skip_asked);
if (!m)
return;
if (m->caller_pinned)
janas_cpus_restore(m->caller_cpus);
if (m->tune_key[0])
janas_tuner_save(&m->tuner, m->tune_key);
if (m->use_count) {
janas_m_profile_write(m);
free(m->use_count);
m->use_count = NULL;
}
janas_m_npu_close(m);
janas_gpu_tok_destroy(m->gpu, m->gtok);
free(m->gtok_layers);
free(m->gblk_layers);
janas_gpu_destroy(m->gpu); /* before the memory it imported; frees
the resident region it allocated */
janas_aligned_free(m->gtok_att_in);
free(m->gtok_ix_rope);
janas_expert_cache_destroy(m->cache);
if (m->mtp_own_cache)
janas_expert_cache_destroy(m->mtp_cache);
janas_attn_destroy(m->attn);
janas_attn_destroy(m->attn_swa);
if (m->own_compute)
janas_pool_destroy(m->compute);
janas_pool_destroy(m->io);
if (!m->resident_gpu)
janas_aligned_free(m->resident);
janas_m_hc_free(m);
janas_m_nh_free(m);
janas_m_qsa_free(m);
if (m->layers)
for (uint32_t l = 0; l < m->n_layer; l++) {
free(m->layers[l].sh_gate);
free(m->layers[l].router_own);
free(m->layers[l].conv_t);
free(m->layers[l].alpha_own);
free(m->layers[l].beta_own);
}
free(m->layers);
float *fl[] = {m->att, m->router, m->rope_cos, m->rope_sin,
m->qgate, m->ba, m->mixed, m->conv_out,
m->beta, m->decay, m->pair_w, m->h};
for (size_t i = 0; i < sizeof(fl) / sizeof(fl[0]); i++)
free(fl[i]);
/* the products' buffers (out_alloc) and the KV cache: aligned */
void *al[] = {m->conv_buf, m->x, m->xn, m->q,
m->kk, m->vv, m->big, m->gate,
m->up, m->dout, m->log_mixed, m->log_beta,
m->log_decay, m->ple, m->ple_g, m->ssm_buf,
m->kcache, m->vcache, m->kscale, m->vscale};
for (size_t i = 0; i < sizeof(al) / sizeof(al[0]); i++)
janas_aligned_free(al[i]);
free(m->kv_off);
free(m->ks_off);
free(m->kv_cap);
free(m->kv_ring);
free(m->kv_swa);
free(m->kv_hd);
free(m->kv_heads);
free(m->rope_cos_swa);
free(m->ple_row);
free(m->ple_q);
free(m->rope_sin_swa);
free(m->xres);
free(m->xbr);
free(m->hlen);
janas_m_ana_free(m);
janas_m_janet_free(m);
janas_m_neurons_free(m);
free(m->xq);
free(m->attq);
free(m->pair_tok);
free(m->pair_exp);
free(m->order);
free(m->glist);
free(m->count);
free(m->xg);
free(m->hq);
free(m->tok_first);
free(m->tok_fill);
free(m->tok_list);
janas_m_assist_free(m);
if (m->mtp_open)
janas_jns_close(&m->mtp_j);
janas_aligned_free(m->mtp_res);
free(m->hid);
free(m->mtp_hid);
free(m->cat);
free(m->catq);
free(m->dh_w);
free(m->dh_in);
free(m->dh_ids);
free(m->dh_logits);
free(m->vs_w);
free(m->vs_ids);
free(m->vs_y);
free(m->mtp_layer.sh_gate);
free(m->mtp_layer.router_own);
free(m->mtp_layer.conv_t);
free(m->mtp_layer.alpha_own);
free(m->mtp_layer.beta_own);
janas_jns_close(&m->j);
free(m);
}
uint32_t janas_llm_model_n_vocab(const struct janas_llm_model *m)
{
return m->n_vocab;
}
uint32_t janas_llm_model_n_embd(const struct janas_llm_model *m)
{
return m->d_model;
}
uint32_t janas_llm_model_experts(const struct janas_llm_model *m,
uint32_t *most)
{
if (most)
*most = m->k;
return m->k_use;
}
int janas_llm_model_set_experts(struct janas_llm_model *m, uint32_t n)
{
if (n == 0)
n = m->k;
if (n > m->k)
return 0;
m->k_use = n;
/* asked for a number: that many, none left out by the engine */
if (m->k_auto) {
m->miss_skip = 0.0f;
m->k_auto = 0;
}
return 1;
}
void janas_llm_model_route_observer(struct janas_llm_model *m,
janas_llm_route_fn fn, void *ctx)
{
m->route_fn = fn;
m->route_ctx = ctx;
}
void janas_llm_model_route_predictor(struct janas_llm_model *m,
janas_llm_route_fn fn, void *ctx,
uint32_t k)
{
m->pred_fn = fn;
m->pred_ctx = ctx;
m->pred_k = k;
}
void janas_llm_model_route_predictor_early(struct janas_llm_model *m, int early)
{
m->pred_early = early;
}
struct janas_gpu *janas_llm_model_gpu(struct janas_llm_model *m)
{
return m->gpu;
}
const char *janas_llm_model_gpu_why(const struct janas_llm_model *m)
{
return m->gpu ? "" : m->gpu_why;
}
uint32_t janas_llm_model_n_ctx(const struct janas_llm_model *m)
{
return m->n_ctx;
}
uint32_t janas_llm_model_n_ctx_train(const struct janas_llm_model *m)
{
return m->n_ctx_train;
}
int janas_llm_model_attn_scores(const struct janas_llm_model *m)
{
return janas_attn_scores_mode(m->attn);
}
int janas_llm_model_attn_scores_auto(const struct janas_llm_model *m)
{
return janas_attn_scores_auto(m->attn);
}
int janas_llm_model_expert_bits(const struct janas_llm_model *m)
{
return 2 * m->exp_level;
}
int janas_llm_model_expert_bits_auto(const struct janas_llm_model *m)
{
return m->exp_auto;
}
void janas_llm_model_head_requant(const struct janas_llm_model *m,
uint64_t *before, uint64_t *after)
{
*before = m->head_before;
*after = m->head_after;
}
int janas_llm_model_head16(const struct janas_llm_model *m, uint64_t *before,
uint64_t *after)
{
*before = m->head16_before;
*after = m->head16_after;
return m->head16;
}
int janas_llm_model_experts_auto(const struct janas_llm_model *m)
{
return m->k_auto;
}
float janas_llm_model_miss_skip(const struct janas_llm_model *m)
{
return m->miss_skip;
}
uint64_t janas_llm_model_warm(const struct janas_llm_model *m, double *seconds)
{
if (seconds)
*seconds = m->warm_seconds;
return m->warm_experts;
}
/* The GPU region of 4 GB at the most: cut at a multiple of unit (an
expert slot), or for the resident region (unit 0) where a tensor
begins, so that no tensor straddles two and each stays in reach */
/*
* Memory the GPU imports, collapsed into huge pages first: the GPU's own
* page tables map it in 2 MB pages only where it is on them and the region
* starts on a 2 MB boundary, and it reads such memory a fifth faster than
* memory in 4 KB pages - 71.6 GB/s against 59.6 on the development laptop,
* where the CPU reads both alike (13 Oct 2026; memory the kernel had already
* put on huge pages, but not from a 2 MB boundary, read as 4 KB pages). A
* request, collapsed once, before the import pins the pages; nothing is
* asked of the kernel while the model answers (see expert_cache.c on huge
* pages asked for on first touch). JANAS_GPU_HUGE=0: not asked.
*/
void janas_m_huge_pages(void *p, uint64_t bytes)
{
#if defined(__linux__)
#ifndef MADV_COLLAPSE
#define MADV_COLLAPSE 25
#endif
const char *e = getenv("JANAS_GPU_HUGE");
if (e && atoi(e) == 0)
return;
uintptr_t a = ((uintptr_t)p + JANAS_HUGE_PAGE - 1) &
~(uintptr_t)(JANAS_HUGE_PAGE - 1);
uintptr_t end = ((uintptr_t)p + bytes) & ~(uintptr_t)(JANAS_HUGE_PAGE - 1);
/* in pieces: the kernel refuses a range holding pages never written
(a cache's slots not yet loaded), and the pieces already on huge
pages cost nothing */
const uintptr_t piece = (uintptr_t)32 << 20;
size_t done = 0, tried = 0;
struct timespec t0, t1;
clock_gettime(CLOCK_MONOTONIC, &t0);
for (uintptr_t at = a; at < end; at += piece, tried++) {
uintptr_t to = at + piece < end ? at + piece : end;
done += madvise((void *)at, to - at, MADV_COLLAPSE) == 0;
}
clock_gettime(CLOCK_MONOTONIC, &t1);
if (getenv("JANAS_GPU_VERBOSE") && tried)
fprintf(stderr,
"huge pages: %.2f GB, %zu of %zu pieces collapsed in %.2f s\n",
(end - a) / 1e9, done, tried,
(t1.tv_sec - t0.tv_sec) + 1e-9 * (t1.tv_nsec - t0.tv_nsec));
#else
(void)p;
(void)bytes;
#endif
}
/* The last tensor boundary of the resident memory past from and at most
room further (from + room if none). */
static uint64_t resident_cut(const struct janas_llm_model *m, uint64_t from,
uint64_t room)
{
uint64_t base = m->j.h.resident_offset, best = from;
for (uint64_t i = 0; i < m->j.h.n_tensor; i++) {
uint64_t at = m->j.tensors[i].offset - base;
if (at > from && at <= from + room && at > best)
best = at;
}
return best > from ? best : from + room;
}
/*
* Each region from a 2 MB boundary (the memory is allocated on one),
* overlapping the one before by what it takes: the GPU maps only such
* regions in huge pages (janas_m_huge_pages). The overlap counts in the
* region's length: from 13 October 2026, when the regions began on 2 MB
* boundaries, until 14 October it did not, and a region past the first
* could come to more than the 4 GiB the GPU takes - Qwen3.6-35B-A3B's
* MXFP4_MOE file, slots of 1,671,168 bytes, the arena's second region 4 GiB
* and 1.9 MB (its Q4_K_M at six bits the third): refused, and the experts
* of a cache holding more than 4 GiB (8 GiB) stayed off the GPU.
*/
uint64_t janas_m_import_next(uint64_t at, uint64_t bytes, uint64_t unit,
int huge, uint64_t *from)
{
*from = huge ? at / JANAS_HUGE_PAGE * JANAS_HUGE_PAGE : at;
uint64_t room = JANAS_M_REGION_MAX - (at - *from);
if (bytes - at <= room)
return bytes;
return unit && room >= unit ? at + room / unit * unit : at + room;
}
int janas_m_gpu_import(struct janas_llm_model *m, const uint8_t *p,
uint64_t bytes, uint64_t unit)
{
int huge =
((uintptr_t)p & (JANAS_HUGE_PAGE - 1)) == 0 &&
!(getenv("JANAS_GPU_HUGE") && atoi(getenv("JANAS_GPU_HUGE")) == 0);
/* the resident region's hole left out (a region after it may start
up to 2 MB into it, its huge page's boundary) */
uint64_t ha = bytes, hb = bytes;
if (p == m->resident && m->res_hole[1] > m->res_hole[0])
ha = m->res_hole[0], hb = m->res_hole[1];
for (uint64_t at = 0; at < bytes;) {
if (at == ha) {
at = hb;
continue;
}
uint64_t lim = at < ha ? ha : bytes;
uint64_t from, end = janas_m_import_next(at, lim, unit, huge, &from);
if (!unit && end < lim)
end = resident_cut(m, at, end - at);
/* whole pages, the last one's length rounded up (the allocations
are) */
uint64_t len = (end - from + 4095) & ~(uint64_t)4095;
if (janas_gpu_import(m->gpu, p + from, len) != 0)
return -1;
at = end;
}
return 0;
}
/*
* The experts on the GPU too: once the cache holds every expert of every
* layer and nothing is being read (a dense model's feed-forwards, or a
* mixture that fits), its arena is shared with the GPU or copied to its
* memory, and the experts' products are split between the two like the
* others. Checked after each pass until then; one change of a slot later
* (a new level of bits) and the GPU's copy is no longer used.
*/
uint32_t janas_llm_prefill_block(const struct janas_jns *j, uint32_t at_least)
{
/* 512 where the GPU takes the feed-forwards whole: a mixture's experts
(gpu.c experts_id) get some 32 tokens each, not 16, and fill its
tiles better - Qwen3-30B-A3B read a prompt of 1024 tokens at 122.7
tokens/s instead of 108.5 (GPU share of the experts' tokens 0.5;
0.4: 112.7, 0.6: 119.2; blocks of 1024: 101); a dense model's one
"expert" too, Qwen3-4B 134.6 instead of 127.0. Not for the others,
whose products the CPU shares: Gemma 4 26B-A4B (experts' down Q5_1,
the rest Q8_0) 63 instead of 68 (9 Oct 2026). Qwen3.5 (qwen35, its
recurrent layers' projections Q5_K): 2B 278.9 -> 318.6, 9B 80.3 ->
86.0, 0.8B 489.0 -> 493.5 (10 Oct 2026). */
/* measured on these architectures only: Qwen3-Next's recurrent layers
and the others keep the block they had */
char arch[32] = "";
janas_jns_meta_str(j, "general.architecture", arch, sizeof(arch));
int fast = janas_gpu_fast_prompt() && j->h.n_expert >= 1 && j->layers &&
(!strcmp(arch, "qwen3") || !strcmp(arch, "qwen3moe") ||
!strcmp(arch, "qwen35"));
for (uint32_t l = 0; fast && l < j->h.n_layer; l++) {
const struct janas_jns_matrix *x = j->layers[l].m;
fast = x[0].type == JANAS_Q4_K && x[1].type == JANAS_Q4_K &&
(x[2].type == JANAS_Q4_K || x[2].type == JANAS_Q6_K ||
x[2].type == JANAS_Q6_K_P);
}
const char *pb = getenv("JANAS_PREFILL_BLOCK");
long v = pb ? atol(pb) : fast ? 512 : 256;
if (v < (long)at_least)
v = (long)at_least;
return v < JANAS_LLM_MAX_BLOCK ? JANAS_LLM_MAX_BLOCK
: v > 4096 ? 4096
: (uint32_t)v;
}
void janas_m_arena_to_gpu(struct janas_llm_model *m)
{
if (!m->gpu || m->gpu_off || !m->cache || m->arena_gpu < 0)
return;
/* a mixture on a card with memory of its own: its experts through the
tier the whole pass keeps there (gpu_token_moe.c), not a copy of the
arena beside it */
if (m->arena_gpu == 0 && janas_m_gpu_tier(m)) {
m->arena_gpu = -1;
if (getenv("JANAS_GPU_VERBOSE"))
fprintf(stderr, "arena: not copied, the experts through the "
"card's tier\n");
return;
}
if (m->arena_gpu == 1) {
if (!m->arena_stale && !m->arena_in_place &&
janas_expert_cache_loads(m->cache) != m->arena_loads)
m->arena_stale = 1;
return;
}
uint64_t bytes, loads = 0;
const uint8_t *a = janas_expert_cache_resident(m->cache, &bytes, &loads);
/*
* A GPU that reads the host's memory in place (an integrated one) needs
* no complete cache: it reads each slot where the CPU does, as loaded,
* and never holds a stale copy. Waiting for every expert of every layer
* keeps a mixture's experts off it for good - Qwen3-30B-A3B's warm-up
* loads the 5,620 of its 6,144 its prompts use, and the rest never come
* (9 Oct 2026). Only the fast prompt gives the GPU experts (gpu.c
* experts_id), so only with it, or with JANAS_GPU_ARENA_INPLACE=1.
*/
int in_place =
!a && (janas_gpu_fast_prompt() || getenv("JANAS_GPU_ARENA_INPLACE")) &&
janas_gpu_shares_host(m->gpu) && !getenv("JANAS_GPU_ALLOC") &&
!getenv("JANAS_GPU_TOKEN");
if (in_place) {
a = janas_expert_cache_arena(m->cache, &bytes);
#ifdef MADV_POPULATE_WRITE
/* every page the arena's own first: a slot never loaded is the
kernel's shared zero page, and each load that replaces one
invalidates the import, pinned again whole (4 GB) at the next
submission - 1.6 s of submissions on a block of 256 tokens of
Qwen3-30B-A3B instead of 0.1 (9 Oct 2026) */
madvise((void *)a, bytes, MADV_POPULATE_WRITE);
#endif
}
if (!a)
return;
uint64_t slot = janas_expert_cache_slot_bytes(m->cache);
uint8_t *own = NULL;
if (getenv("JANAS_GPU_EXPERTS") && atoi(getenv("JANAS_GPU_EXPERTS")) == 0)
m->arena_gpu = -1;
else if ((getenv("JANAS_GPU_ALLOC") || getenv("JANAS_GPU_TOKEN")) &&
janas_gpu_shares_host(m->gpu) &&
(own = janas_gpu_alloc(m->gpu, bytes)) &&
janas_expert_cache_rebase(m->cache, own) == 0) {
/* moved into the GPU driver's memory, which it reads faster than
imported pages; the CPU reads the same memory */
janas_gpu_flush(m->gpu, own, bytes);
m->arena_gpu = 1;
m->arena_loads = loads;
m->arena_in_place = in_place;
if (getenv("JANAS_GPU_VERBOSE"))
fprintf(stderr, "arena: %.1f GB in the GPU driver's memory%s\n",
bytes / 1e9, in_place ? " (in place)" : "");
} else if (janas_m_huge_pages((void *)a, bytes),
janas_m_gpu_import(m, a, bytes, slot) == 0) {
m->arena_gpu = 1;
m->arena_loads = loads;
m->arena_in_place = in_place;
if (getenv("JANAS_GPU_VERBOSE"))
fprintf(stderr, "arena: %.1f GB imported%s\n", bytes / 1e9,
in_place ? " (in place)" : "");
} else {
m->arena_gpu = -1; /* no room for it: the CPU goes on alone */
if (getenv("JANAS_GPU_VERBOSE"))
fprintf(stderr,
"arena: %.1f GB not imported: the experts stay "
"on the CPU\n",
bytes / 1e9);
}
}
int janas_llm_model_settling(const struct janas_llm_model *m)
{
return m->cache && janas_expert_cache_warming(m->cache);
}
uint64_t janas_llm_model_warm_done(const struct janas_llm_model *m)
{
uint64_t done = 0;
if (m->cache)
janas_expert_cache_warm_progress(m->cache, &done, NULL);
return done;
}
struct janas_expert_cache *janas_llm_model_cache(struct janas_llm_model *m)
{
return m->cache;
}