37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
/*
* gguf2jns - converts a GGUF model into the Janas-LLM file format (JNS, see
* src/llm/jns.h).
*
* Routed experts are rewritten one slot per (layer, expert), with gate, up
* and down back to back, so the engine can read an expert with a single
* aligned O_DIRECT request. Every other tensor goes, unchanged, into the
* resident region. A dense model has no experts to route: its feed-forward
* becomes the single slot of the layer, written in the same way.
*
* A feed-forward whose width is not a multiple of 256 (Gemma 4's 26B: 704
* and 2112) is widened with zeros to the next one - rows of zero bytes in
* gate and up, a zero tail on each row of down: every weight type reads zero
* bytes as zero weights, so the products are the same, and the engine's
* activations, quantized 256 at a time, fit. Qwen3.8-Flash-Next's experts
* (qwen4exp, 640 wide) keep their width: the engine reads rows that end
* inside a group of 256 in the types stored in blocks of 32 or 64
* (janas_dot_rows_n), the only ones such a down can have, and the zeros
* would be a fifth more bytes read at every token. Gate and up fused in
* one tensor (ffn_gate_up_exps) are split into the two matrices of the
* slot.
*
* A resident tensor in MXFP4 (unsloth's Qwen3-Coder-Next files have their
* shared experts so, issue #18) is written as Q8_0, which holds it exactly:
* an MXFP4 weight is an integer of -12 to 12 times a power of two, a block's
* one (E8M0, 2^(e - 128)), and so is Q8_0's with the power as its fp16
* scale. Only a block whose power is below fp16's least, 2^-24, and whose
* weights are so below 3.6e-7, is written as zeros (counted). Routed
* experts in MXFP4 (the MXFP4_MOE files) stay MXFP4, which the engine reads
* as it is: as Q8_0 they would read twice the bytes at every token.
*
* Llama's queries and keys (general.architecture "llama") are written in the
* row order of the original checkpoint: llama.cpp's converter permutes the
* rows of each head, [2][half] to [half][2], for its rotation of adjacent
* pairs, and Janas-LLM rotates halves, as Qwen's files have it. The rows are
* moved whole, so any type of weight moves exactly.
*
* A model split into several GGUF files (name-00001-of-0000N.gguf) is read
* from all of them: give the first. Data is copied in streaming; the model
* is never held in memory.
*
* The file written is the one the Python converter it replaced wrote, byte
* for byte (JNS version 2): the fingerprints in MODELS.md hold for both.
*
* Usage: gguf2jns <model.gguf> <model.jns>
*/
#include <errno.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include "llm/gguf.h"
#include "llm/jns.h"
#include "llm/quant.h"
#define ALIGN 4096
#define MATRIX_ALIGN 256
#define COPY_CHUNK ((size_t)16 << 20)
#define MAX_PARTS 256
#define GGUF_MXFP4 39 /* ggml's type number */
#define MXFP4_BLOCK JANAS_MXFP4_BLOCK
#define Q8_0_BLOCK sizeof(struct janas_block_q8_0)
static uint64_t align_up(uint64_t x, uint64_t a)
{
return (x + a - 1) / a * a;
}
static _Noreturn void die(const char *fmt, const char *arg)
{
fprintf(stderr, "gguf2jns: ");
fprintf(stderr, fmt, arg);
fputc('\n', stderr);
exit(1);
}
/* A tensor of the model: which part of a split model, and where. */
struct tensor {
const struct janas_gguf_tensor *t;
int part;
uint64_t base; /* the part's data start */
int expert; /* one of the experts' matrices */
uint64_t new_offset;
/* a resident matrix widened with zeros: its rows, then as written */
uint64_t rows, rb, new_rows, new_rb, new_dims[2], new_bytes;
uint32_t new_type; /* MXFP4 is written as Q8_0 */
uint32_t unperm; /* Llama's attn_q, attn_k: their heads, rows to put
back in the checkpoint's order */
};
/* A matrix of a slot: from which tensor, where in an expert's part of it,
its rows and row bytes, and as written. */
struct src_matrix {
struct tensor *t;
uint64_t off, per; /* in an expert's bytes of t, and those bytes */
uint64_t rows, rb, new_rows, new_rb;
};
struct part {
struct janas_gguf g;
int fd;
};
static struct part parts[MAX_PARTS];
static int n_parts;
static struct tensor *tensors;
static size_t n_tensors;
static uint64_t copied;
static int out_fd;
static uint8_t *buf;
static char *str_dup(const struct janas_gguf_str *s)
{
char *d = malloc(s->n + 1);
if (!d)
die("%s", "out of memory");
memcpy(d, s->p, s->n);
d[s->n] = 0;
return d;
}
static uint64_t need_uint(const struct janas_gguf *g, const char *key)
{
uint64_t v;
if (janas_gguf_uint(g, key, &v) != 0)
die("metadata %s missing, or not a number", key);
return v;
}
/* An integer, or the largest of an array of them (nemotron_h_moe's
expert_used_count, one a layer); 0 if neither. */
static uint64_t uint_or_max(const struct janas_gguf *g, const char *key)
{
uint64_t v;
if (janas_gguf_uint(g, key, &v) == 0)
return v;
for (uint64_t i = 0; i < g->n_kv; i++) {
const struct janas_gguf_kv *kv = &g->kv[i];
if (kv->key.n != strlen(key) || memcmp(kv->key.p, key, kv->key.n) ||
kv->type != 9)
continue;
uint32_t et;
uint64_t n, best = 0;
memcpy(&et, kv->val, 4);
memcpy(&n, kv->val + 4, 8);
if (et != 4 && et != 5)
return 0;
for (uint64_t k = 0; k < n; k++) {
uint32_t x;
memcpy(&x, kv->val + 12 + 4 * k, 4);
if (x > best && !(et == 5 && (int32_t)x < 0))
best = x;
}
return best;
}
return 0;
}
/* The tensor of this name in any part; the last part that has it wins, as
the dictionary of the Python script had it. */
static struct tensor *find(const char *name)
{
size_t n = strlen(name);
for (size_t i = n_tensors; i-- > 0;)
if (tensors[i].t->name.n == n &&
memcmp(tensors[i].t->name.p, name, n) == 0)
return &tensors[i];
return NULL;
}
static void open_part(const char *path)
{
if (n_parts == MAX_PARTS)
die("%s: too many parts", path);
struct part *p = &parts[n_parts];
char err[512];
if (janas_gguf_open(path, &p->g, err, sizeof(err)) != 0)
die("%s", err);
p->fd = open(path, O_RDONLY | O_CLOEXEC);
if (p->fd < 0)
die("%s: cannot open", path);
struct tensor *grown =
realloc(tensors, (n_tensors + p->g.n_tensors) * sizeof(*grown));
if (!grown && p->g.n_tensors)
die("%s", "out of memory");
tensors = grown;
for (uint64_t i = 0; i < p->g.n_tensors; i++)
tensors[n_tensors++] = (struct tensor){
.t = &p->g.t[i], .part = n_parts, .base = p->g.data_start};
n_parts++;
}
/* Copies n bytes of a tensor's data, from its offset src, to dst in the
output; with crc, their CRC-32 carried on. */
static uint32_t copy(const struct tensor *t, uint64_t src, uint64_t n,
uint64_t dst, uint32_t crc, int with_crc)
{
int fd = parts[t->part].fd;
uint64_t from = t->base + src;
while (n) {
size_t k = n < COPY_CHUNK ? (size_t)n : COPY_CHUNK;
ssize_t r = pread(fd, buf, k, (off_t)from);
if (r <= 0)
die("%s", "unexpected end of the GGUF file");
if (with_crc)
crc = janas_crc32(crc, buf, (size_t)r);
for (ssize_t w = 0; w < r;) {
ssize_t x = pwrite(out_fd, buf + w, (size_t)(r - w),
(off_t)(dst + (uint64_t)w));
if (x <= 0)
die("write failed: %s", strerror(errno));
w += x;
}
from += (uint64_t)r;
dst += (uint64_t)r;
n -= (uint64_t)r;
copied += (uint64_t)r;
}
return crc;
}
/* The CRC-32 of n zero bytes carried on. */
static uint32_t crc_zeros(uint32_t crc, uint64_t n)
{
static const uint8_t zeros[4096];
while (n) {
size_t k = n < sizeof(zeros) ? (size_t)n : sizeof(zeros);
crc = janas_crc32(crc, zeros, k);
n -= k;
}
return crc;
}
static uint8_t *obuf; /* rows widened, before they are written */
/*
* rows rows of rb bytes at src of t, to dst as new_rows rows of new_rb
* bytes: each row's tail and the extra rows zero (the output is created
* zero-filled, so only the CRC has to count them).
*/
static uint32_t copy_rows(const struct tensor *t, uint64_t src, uint64_t rows,
uint64_t rb, uint64_t new_rows, uint64_t new_rb,
uint64_t dst, uint32_t crc, int with_crc)
{
if (new_rb == rb) {
crc = copy(t, src, rows * rb, dst, crc, with_crc);
} else {
/* a row at a time at the least: it must fit the buffers */
if (new_rb < rb || new_rb > COPY_CHUNK)
die("%s", "a row longer than 16 MiB: not one this converter "
"rewrites");
uint64_t per = COPY_CHUNK / new_rb ? COPY_CHUNK / new_rb : 1;
int fd = parts[t->part].fd;
for (uint64_t r0 = 0; r0 < rows; r0 += per) {
uint64_t k = rows - r0 < per ? rows - r0 : per;
if (pread(fd, buf, k * rb, (off_t)(t->base + src + r0 * rb)) !=
(ssize_t)(k * rb))
die("%s", "unexpected end of the GGUF file");
memset(obuf, 0, k * new_rb);
for (uint64_t r = 0; r < k; r++)
memcpy(obuf + r * new_rb, buf + r * rb, rb);
if (with_crc)
crc = janas_crc32(crc, obuf, k * new_rb);
if (pwrite(out_fd, obuf, k * new_rb, (off_t)(dst + r0 * new_rb)) !=
(ssize_t)(k * new_rb))
die("write failed: %s", strerror(errno));
copied += k * rb;
}
}
if (with_crc)
crc = crc_zeros(crc, (new_rows - rows) * new_rb);
return crc;
}
/* The rows of t, heads of rows / heads each, from llama.cpp's order - row
2 i + j of a head - back to the checkpoint's, row j * half + i. */
static void copy_unpermuted(const struct tensor *t, uint64_t dst)
{
uint64_t rows = t->t->dims[1], rb = t->t->bytes / rows;
uint64_t hd = rows / t->unperm, half = hd / 2;
uint8_t *in = malloc(t->t->bytes), *outb = malloc(t->t->bytes);
if (!in || !outb)
die("%s", "out of memory");
if (pread(parts[t->part].fd, in, t->t->bytes,
(off_t)(t->base + t->t->offset)) != (ssize_t)t->t->bytes)
die("%s", "unexpected end of the GGUF file");
for (uint64_t h = 0; h < t->unperm; h++)
for (uint64_t j = 0; j < 2; j++)
for (uint64_t i = 0; i < half; i++)
memcpy(outb + (h * hd + j * half + i) * rb,
in + (h * hd + 2 * i + j) * rb, rb);
if (pwrite(out_fd, outb, t->t->bytes, (off_t)dst) != (ssize_t)t->t->bytes)
die("write failed: %s", strerror(errno));
copied += t->t->bytes;
free(in);
free(outb);
}
static size_t tiny_blocks; /* MXFP4 blocks below fp16's scales */
/* n bytes of MXFP4 at src of t, to dst as Q8_0 (twice the bytes). */
static void copy_mxfp4(const struct tensor *t, uint64_t src, uint64_t n,
uint64_t dst)
{
int fd = parts[t->part].fd;
uint64_t per = COPY_CHUNK / MXFP4_BLOCK * MXFP4_BLOCK;
for (uint64_t at = 0; at < n; at += per) {
uint64_t k = n - at < per ? n - at : per;
if (pread(fd, buf, k, (off_t)(t->base + src + at)) != (ssize_t)k)
die("%s", "unexpected end of the GGUF file");
uint64_t nb = k / MXFP4_BLOCK;
if (janas_mxfp4_to_q8_0(buf, (struct janas_block_q8_0 *)obuf, nb,
&tiny_blocks) != 0)
die("%s", "an MXFP4 scale beyond fp16's (NaN, or 2^16 and more)");
if (pwrite(out_fd, obuf, nb * Q8_0_BLOCK,
(off_t)(dst + at / MXFP4_BLOCK * Q8_0_BLOCK)) !=
(ssize_t)(nb * Q8_0_BLOCK))
die("write failed: %s", strerror(errno));
copied += k;
}
}
/* A width rounded up to a multiple of 256 (the engine's activations). */
static uint64_t wide(uint64_t n)
{
return (n + 255) / 256 * 256;
}
static int by_place(const void *a, const void *b)
{
const struct tensor *x = *(const struct tensor *const *)a;
const struct tensor *y = *(const struct tensor *const *)b;
if (x->part != y->part)
return x->part < y->part ? -1 : 1;
return x->t->offset < y->t->offset ? -1 : x->t->offset > y->t->offset;
}
int main(int argc, char **argv)
{
if (argc != 3) {
fprintf(stderr, "usage: gguf2jns <model.gguf> <model.jns>\n"
" a model split into name-0000N-of-0000M.gguf is "
"read from all its parts: give the first\n");
return 2;
}
const char *src = argv[1], *dst = argv[2];
open_part(src);
const struct janas_gguf *g0 = &parts[0].g;
/* the other parts of a split model */
uint64_t count = 1, no = 0;
janas_gguf_uint(g0, "split.count", &count);
if (count > 1) {
janas_gguf_uint(g0, "split.no", &no);
size_t n = strlen(src);
const char *tail =
n >= 20 ? src + n - 20 : NULL; /* -00001-of-0000N.gguf */
if (!tail || no != 0 || count > MAX_PARTS ||
strncmp(tail, "-00001-of-", 10) != 0 ||
strcmp(tail + 15, ".gguf") != 0)
die("%s: give the first file of the split model", src);
for (uint64_t i = 2; i <= count; i++) {
char *part = malloc(n + 1);
if (!part)
die("%s", "out of memory");
snprintf(part, n + 1, "%.*s-%05llu-of-%05llu.gguf", (int)(n - 20),
src, (unsigned long long)i, (unsigned long long)count);
open_part(part);
free(part);
}
}
struct janas_gguf_str arch_s;
if (janas_gguf_string(g0, "general.architecture", &arch_s) != 0)
die("%s", "general.architecture missing");
char *arch = str_dup(&arch_s);
char key[160];
snprintf(key, sizeof(key), "%s.block_count", arch);
uint64_t n_layer = need_uint(g0, key);
/* a dense model is a mixture with one expert and no router: the engine
gives it weight 1 through the same softmax, and the feed-forward of a
layer becomes its only slot */
snprintf(key, sizeof(key), "%s.expert_count", arch);
uint64_t n_expert = 1, n_used = 1;
int dense = janas_gguf_uint(g0, key, &n_expert) != 0;
if (dense)
n_expert = 1;
else {
snprintf(key, sizeof(key), "%s.expert_used_count", arch);
n_used = uint_or_max(g0, key);
if (n_used == 0)
die("metadata %s missing, or not a number", key);
}
snprintf(key, sizeof(key), "%s.embedding_length", arch);
uint64_t d_model = need_uint(g0, key);
if (n_layer == 0 || n_layer > JANAS_JNS_MAX_LAYERS || n_expert == 0 ||
n_expert > 65536)
die("%s: layers or experts out of range", src);
/* the experts' width as it is (see the top of the file) */
int keep_width = strcmp(arch, "qwen4exp") == 0;
uint64_t llama_heads = 0, llama_kv = 0;
if (strcmp(arch, "llama") == 0) {
llama_heads = need_uint(g0, "llama.attention.head_count");
llama_kv = need_uint(g0, "llama.attention.head_count_kv");
}
/* nemotron_h_moe: a layer is a Mamba-2 mixer, an attention or a
mixture, and only the mixtures have experts - each its own number of
them (the router's rows) and no gate (relu^2 on up). Such a layer
without experts has an empty slot (no matrices, size 0), a missing
gate an empty matrix, and the table is as wide as the layer with the
most experts: the others' slots past their own are absent (offset
0), never asked for. */
int sparse = strcmp(arch, "nemotron_h_moe") == 0;
uint64_t *layer_ne = calloc(n_layer, sizeof(*layer_ne));
if (!layer_ne)
die("%s", "out of memory");
if (sparse) {
n_expert = 0;
for (uint64_t l = 0; l < n_layer; l++) {
snprintf(key, sizeof(key), "blk.%llu.ffn_up_exps.weight",
(unsigned long long)l);
struct tensor *t = find(key);
layer_ne[l] = t && t->t->n_dims > 2 ? t->t->dims[2] : 0;
if (layer_ne[l] > n_expert)
n_expert = layer_ne[l];
}
if (n_expert == 0 || n_expert > 65536)
die("%s: no layer with experts", src);
for (uint64_t l = 0; l < n_layer; l++)
if (layer_ne[l] && layer_ne[l] < n_used)
die("%s: a layer has fewer experts than are used", src);
} else
for (uint64_t l = 0; l < n_layer; l++)
layer_ne[l] = n_expert;
if (n_used > n_expert)
die("%s: more experts used than there are", src);
/* expert geometry per layer */
struct janas_jns_layer_v2 *layers = calloc(n_layer, sizeof(*layers));
struct src_matrix(*mats)[3] = calloc(n_layer, sizeof(*mats));
if (!layers || !mats)
die("%s", "out of memory");
static const char *const part_name[3] = {"gate", "up", "down"};
for (uint64_t l = 0; l < n_layer; l++) {
uint64_t off = 0;
if (layer_ne[l] == 0)
continue; /* no experts: an empty slot */
for (int p = 0; p < 3; p++) {
snprintf(key, sizeof(key),
dense ? "blk.%llu.ffn_%s.weight"
: "blk.%llu.ffn_%s_exps.weight",
(unsigned long long)l, part_name[p]);
struct tensor *t = find(key);
int half = 0; /* gate and up fused: this one's half of the rows */
if (!t && !dense && p < 2) {
snprintf(key, sizeof(key), "blk.%llu.ffn_gate_up_exps.weight",
(unsigned long long)l);
t = find(key);
half = 1;
}
if (!t && sparse && p == 0)
continue; /* no gate: an empty matrix */
if (!t)
die("%s missing: only models with routed experts in every "
"layer, or dense ones, are supported",
key);
uint64_t ne = t->t->n_dims > 2 ? t->t->dims[2] : 1;
if (ne != layer_ne[l] || t->t->bytes % ne ||
t->t->dims[0] > UINT32_MAX || t->t->dims[1] > UINT32_MAX ||
t->t->dims[3] != 1 || (half && t->t->dims[1] % 2))
die("%s: unexpected shape", key);
uint64_t per = t->t->bytes / ne;
uint64_t rows = t->t->dims[1] / (half ? 2 : 1);
uint64_t cols = t->t->dims[0];
if (per % t->t->dims[1])
die("%s: rows of unequal size", key);
uint64_t rb = per / t->t->dims[1];
/* widened to a multiple of 256: the rows of gate and up, the
row of down */
uint64_t new_rows = p < 2 && !keep_width ? wide(rows) : rows;
uint64_t new_cols = p < 2 || keep_width ? cols : wide(cols);
if (new_cols != cols && (rb * new_cols) % cols)
die("%s: cannot widen this type", key);
if (keep_width &&
janas_qtype_row_bytes((int)t->t->type, cols) != rb)
die("%s: a row the engine cannot read at this width", key);
uint64_t new_rb = rb * new_cols / cols;
off = align_up(off, MATRIX_ALIGN);
layers[l].m[p] =
(struct janas_jns_matrix){.type = t->t->type,
.rows = (uint32_t)new_rows,
.cols = (uint32_t)new_cols,
.offset = off,
.bytes = new_rows * new_rb};
off += new_rows * new_rb;
t->expert = 1;
mats[l][p] =
(struct src_matrix){.t = t,
.off = p == 1 && half ? rows * rb : 0,
.per = per,
.rows = rows,
.rb = rb,
.new_rows = new_rows,
.new_rb = new_rb};
}
layers[l].slot_bytes = align_up(off, ALIGN);
}
/* the resident tensors, in the order of the files */
struct tensor **resident = malloc((n_tensors + 1) * sizeof(*resident));
size_t n_res = 0;
if (!resident)
die("%s", "out of memory");
for (size_t i = 0; i < n_tensors; i++) {
/* a name some later part has too counts once, the later one */
const struct janas_gguf_str *nm = &tensors[i].t->name;
char *name = str_dup(nm);
int shadowed = find(name) != &tensors[i];
free(name);
if (!tensors[i].expert && !shadowed)
resident[n_res++] = &tensors[i];
}
qsort(resident, n_res, sizeof(*resident), by_place);
/* a dense feed-forward beside the experts (Gemma 4's 26B), widened as
theirs */
for (size_t i = 0; i < n_res; i++) {
struct tensor *t = resident[i];
char *name = str_dup(&t->t->name);
size_t nl = strlen(name);
int gu = nl > 16 && (!strcmp(name + nl - 16, ".ffn_gate.weight") ||
!strcmp(name + nl - 14, ".ffn_up.weight"));
int dn = nl > 16 && !strcmp(name + nl - 16, ".ffn_down.weight");
t->new_bytes = t->t->bytes;
t->new_dims[0] = t->t->dims[0];
t->new_dims[1] = t->t->dims[1];
t->new_type = t->t->type;
if (llama_heads && nl > 14 &&
(!strcmp(name + nl - 14, ".attn_q.weight") ||
!strcmp(name + nl - 14, ".attn_k.weight"))) {
t->unperm =
(uint32_t)(name[nl - 8] == 'q' ? llama_heads : llama_kv);
if (t->t->n_dims != 2 || t->t->dims[1] % (2 * t->unperm) ||
t->t->bytes % t->t->dims[1])
die("%s: unexpected shape", name);
}
if (t->t->type == GGUF_MXFP4) {
if (t->t->bytes % MXFP4_BLOCK || t->t->dims[0] % 32 ||
(t->t->n_dims == 2 && (gu || dn) &&
(wide(t->t->dims[0]) != t->t->dims[0] ||
wide(t->t->dims[1]) != t->t->dims[1])))
die("%s: an MXFP4 tensor this converter cannot rewrite", name);
t->new_type = JANAS_Q8_0;
t->new_bytes = t->t->bytes / MXFP4_BLOCK * Q8_0_BLOCK;
free(name);
continue;
}
free(name);
if (t->t->n_dims != 2 || !(gu || dn))
continue;
uint64_t rows = t->t->dims[1], cols = t->t->dims[0];
if (t->t->bytes % rows)
die("%s", "a feed-forward matrix with rows of unequal size");
t->rows = rows;
t->rb = t->t->bytes / rows;
t->new_rows = gu ? wide(rows) : rows;
t->new_dims[1] = t->new_rows;
t->new_dims[0] = gu ? cols : wide(cols);
if ((t->rb * t->new_dims[0]) % cols)
die("%s", "a feed-forward matrix whose type cannot be widened");
t->new_rb = t->rb * t->new_dims[0] / cols;
t->new_bytes = t->new_rows * t->new_rb;
}
/* layout */
uint64_t metadata_bytes = 8 + (g0->kv_end - g0->kv_start);
uint64_t layer_table = ALIGN;
uint64_t expert_table =
layer_table + sizeof(struct janas_jns_layer_v2) * n_layer;
uint64_t expert_sums = expert_table + 8 * n_layer * n_expert;
uint64_t tensor_dir = expert_sums + 8 * n_layer * n_expert;
uint64_t metadata_offset =
align_up(tensor_dir + sizeof(struct janas_jns_tensor) * n_res, ALIGN);
uint64_t resident_offset =
align_up(metadata_offset + metadata_bytes, ALIGN);
uint64_t pos = resident_offset;
for (size_t i = 0; i < n_res; i++) {
resident[i]->new_offset = pos;
pos = align_up(pos + resident[i]->new_bytes, ALIGN);
}
uint64_t resident_bytes = pos - resident_offset;
uint64_t experts_offset = pos;
uint64_t *slot_off = malloc(8 * n_layer * n_expert);
uint64_t *sums = malloc(8 * n_layer * n_expert);
if (!slot_off || !sums)
die("%s", "out of memory");
for (uint64_t l = 0; l < n_layer; l++)
for (uint64_t e = 0; e < n_expert; e++) {
/* an absent slot: offset 0 */
slot_off[l * n_expert + e] = e < layer_ne[l] ? pos : 0;
if (e < layer_ne[l])
pos += layers[l].slot_bytes;
}
uint64_t experts_bytes = pos - experts_offset;
uint64_t file_bytes = pos;
char what[96];
if (dense)
snprintf(what, sizeof(what), "dense feed-forward");
else
snprintf(what, sizeof(what), "%llu experts (k %llu)",
(unsigned long long)n_expert, (unsigned long long)n_used);
printf("%s: %llu layers, %s, d_model %llu; %zu resident tensors %.2f GB, "
"experts %.2f GB\n",
arch, (unsigned long long)n_layer, what, (unsigned long long)d_model,
n_res, resident_bytes / 1e9, experts_bytes / 1e9);
out_fd = open(dst, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644);
if (out_fd < 0 || ftruncate(out_fd, (off_t)file_bytes) != 0)
die("%s: cannot write", dst);
buf = malloc(COPY_CHUNK);
obuf = malloc(COPY_CHUNK * 2);
if (!buf || !obuf)
die("%s", "out of memory");
for (size_t i = 0; i < n_res; i++) {
const struct tensor *t = resident[i];
if (t->unperm)
copy_unpermuted(t, t->new_offset);
else if (t->new_type != t->t->type)
copy_mxfp4(t, t->t->offset, t->t->bytes, t->new_offset);
else if (t->new_bytes != t->t->bytes)
copy_rows(t, t->t->offset, t->rows, t->rb, t->new_rows, t->new_rb,
t->new_offset, 0, 0);
else
copy(t, t->t->offset, t->t->bytes, t->new_offset, 0, 0);
}
for (uint64_t l = 0; l < n_layer; l++) {
for (uint64_t e = 0; e < n_expert; e++) {
uint64_t slot = slot_off[l * n_expert + e];
uint32_t crc = 0;
uint64_t end = 0;
if (e >= layer_ne[l]) {
sums[l * n_expert + e] = 0;
continue;
}
for (int p = 0; p < 3; p++) {
const struct janas_jns_matrix *m = &layers[l].m[p];
if (m->type == 0)
continue; /* an empty matrix */
if (m->offset > end) /* padding inside the slot is zero */
crc = crc_zeros(crc, m->offset - end);
const struct src_matrix *s = &mats[l][p];
crc = copy_rows(s->t, s->t->t->offset + e * s->per + s->off,
s->rows, s->rb, s->new_rows, s->new_rb,
slot + m->offset, crc, 1);
end = m->offset + m->bytes;
}
crc = crc_zeros(crc, layers[l].slot_bytes - end);
sums[l * n_expert + e] = crc;
}
printf("\r layer %llu/%llu, %.1f GB copied", (unsigned long long)l + 1,
(unsigned long long)n_layer, copied / 1e9);
fflush(stdout);
}
printf("\n");
if (tiny_blocks)
printf("MXFP4: %llu blocks below fp16's least scale written as zeros "
"(their weights under 3.6e-7)\n",
(unsigned long long)tiny_blocks);
/* tables, in file order, and their checksum */
uint64_t tbl_len =
tensor_dir + sizeof(struct janas_jns_tensor) * n_res - layer_table;
uint8_t *tbl = calloc(1, tbl_len);
if (!tbl)
die("%s", "out of memory");
memcpy(tbl, layers, sizeof(*layers) * n_layer);
memcpy(tbl + (expert_table - layer_table), slot_off,
8 * n_layer * n_expert);
memcpy(tbl + (expert_sums - layer_table), sums, 8 * n_layer * n_expert);
for (size_t i = 0; i < n_res; i++) {
const struct janas_gguf_tensor *t = resident[i]->t;
struct janas_jns_tensor d = {0};
if (t->name.n >= sizeof(d.name)) {
char *name = str_dup(&t->name);
die("tensor name too long: %s", name);
}
memcpy(d.name, t->name.p, t->name.n);
d.type = resident[i]->new_type;
d.n_dims = t->n_dims;
memcpy(d.dims, t->dims, sizeof(d.dims));
d.dims[0] = resident[i]->new_dims[0];
d.dims[1] = resident[i]->new_dims[1];
d.offset = resident[i]->new_offset;
d.bytes = resident[i]->new_bytes;
memcpy(tbl + (tensor_dir - layer_table) + i * sizeof(d), &d, sizeof(d));
}
/* copied as it is, and the JNS reader takes no array of arrays */
for (uint64_t i = 0; i < g0->n_kv; i++) {
uint32_t et;
if (g0->kv[i].type == 9 && (memcpy(&et, g0->kv[i].val, 4), et == 9))
die("%s: an array of arrays in the metadata, which a JNS file "
"cannot hold",
src);
}
uint8_t *meta = malloc(metadata_bytes);
if (!meta)
die("%s", "out of memory");
memcpy(meta, &g0->n_kv, 8);
memcpy(meta + 8, g0->head + g0->kv_start, metadata_bytes - 8);
if (pwrite(out_fd, tbl, tbl_len, (off_t)layer_table) != (ssize_t)tbl_len ||
pwrite(out_fd, meta, metadata_bytes, (off_t)metadata_offset) !=
(ssize_t)metadata_bytes)
die("write failed: %s", strerror(errno));
struct janas_jns_header h = {0};
memcpy(h.magic, JANAS_JNS_MAGIC, 8);
h.version = 2;
h.header_bytes = ALIGN;
h.file_bytes = file_bytes;
memcpy(h.arch, arch, strlen(arch) < 31 ? strlen(arch) : 31);
h.n_layer = (uint32_t)n_layer;
h.n_expert = (uint32_t)n_expert;
h.n_expert_used = (uint32_t)n_used;
h.d_model = (uint32_t)d_model;
h.layer_table_offset = layer_table;
h.expert_table_offset = expert_table;
h.expert_sums_offset = expert_sums;
h.tensor_dir_offset = tensor_dir;
h.n_tensor = n_res;
h.metadata_offset = metadata_offset;
h.metadata_bytes = metadata_bytes;
h.resident_offset = resident_offset;
h.resident_bytes = resident_bytes;
h.experts_offset = experts_offset;
h.experts_bytes = experts_bytes;
h.tables_checksum = janas_fnv1a(tbl, tbl_len, JANAS_FNV_INIT);
h.metadata_checksum = janas_crc32(0, meta, metadata_bytes);
h.header_checksum = janas_fnv1a(&h, sizeof(h), JANAS_FNV_INIT);
if (pwrite(out_fd, &h, sizeof(h), 0) != (ssize_t)sizeof(h) ||
close(out_fd) != 0)
die("write failed: %s", strerror(errno));
printf("written %s: %.2f GB\n", dst, file_bytes / 1e9);
free(meta);
free(tbl);
free(buf);
free(obuf);
free(slot_off);
free(sums);
free(resident);
free(mats);
free(layers);
free(layer_ne);
free(arch);
free(tensors);
for (int i = 0; i < n_parts; i++) {
close(parts[i].fd);
janas_gguf_free(&parts[i].g);
}
return 0;
}