37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
/*
* speaker.c - WeSpeaker's ResNet34 speaker embedding as its ONNX model
* computes it: Kaldi's log mel filter banks (torchaudio's kaldi.fbank: 25
* ms frames every 10 ms, DC removed, pre-emphasis 0.97, Hamming window,
* 512-point power spectrum, 80 triangular bands on Kaldi's mel scale from
* 20 Hz to Nyquist, the samples scaled to 16 bits), their mean over the
* window taken off; a 3x3 convolution to 32 channels, then 3, 4, 6 and 3
* residual blocks of 32, 64, 128 and 256 channels (batch norms folded in,
* strides of 2 and 1x1 shortcuts where the size halves), the frequency and
* channel axes kept, the mean and the standard deviation over time, their
* projection to 256 less the embeddings' mean. The convolutions as tiled
* products of patches (im2col) on the vision encoders' kernels.
*/
#include "speaker.h"
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include "common/audio_fft.h"
#include "common/pool.h"
#include "common/sysinfo.h"
#include "jns.h"
#include "vision_kernels.h"
#define MELS 80
#define NCONV 36
#define DIM 256
struct janas_speaker {
struct janas_jns j;
uint8_t *data;
const uint16_t *w[NCONV];
const float *b[NCONV];
int kernel[NCONV], stride[NCONV], cin[NCONV], cout[NCONV];
const uint16_t *proj;
const float *proj_b, *mean;
struct janas_pool *pool;
};
static const void *tensor(struct janas_speaker *s, const char *name,
uint32_t type, uint64_t count)
{
const struct janas_jns_tensor *t = janas_jns_tensor(&s->j, name);
if (!t || t->type != type || t->bytes != count * (type ? 2 : 4))
return NULL;
return s->data + (t->offset - s->j.h.resident_offset);
}
void janas_speaker_close(struct janas_speaker *s)
{
if (!s)
return;
if (s->pool)
janas_pool_destroy(s->pool);
free(s->data);
janas_jns_close(&s->j);
free(s);
}
struct janas_speaker *janas_speaker_open(const char *path, int n_threads,
char *err, size_t err_len)
{
static const int CH[NCONV] = {32, 32, 32, 32, 32, 32, 32, 64, 64,
64, 64, 64, 64, 64, 64, 64, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 256, 256, 256, 256, 256, 256, 256};
struct janas_speaker *s = calloc(1, sizeof *s);
if (!s) {
snprintf(err, err_len, "out of memory");
return NULL;
}
s->j.fd = -1;
if (janas_jns_open(path, &s->j, err, err_len) != 0) {
free(s);
return NULL;
}
const char *why = NULL;
s->data = malloc(s->j.h.resident_bytes ? s->j.h.resident_bytes : 1);
size_t got = 0;
while (s->data && got < s->j.h.resident_bytes) {
ssize_t k = pread(s->j.fd, s->data + got, s->j.h.resident_bytes - got,
(off_t)(s->j.h.resident_offset + got));
if (k <= 0)
break;
got += (size_t)k;
}
if (!s->data || got != s->j.h.resident_bytes)
why = "cannot read the weights";
const int32_t *ks = NULL, *ss = NULL;
uint64_t nk = 0, ns = 0;
if (!why && (janas_jns_meta_ints(&s->j, "spk.kernels", &nk, &ks) ||
janas_jns_meta_ints(&s->j, "spk.strides", &ns, &ss) ||
nk != NCONV || ns != NCONV))
why = "not a WeSpeaker ResNet34 model (made by wespeaker2jns)";
for (int i = 0; i < NCONV && !why; i++) {
s->kernel[i] = ks[i];
s->stride[i] = ss[i];
s->cout[i] = CH[i];
/* the input: the block's, or the layer before's for its first */
s->cin[i] = i == 0 ? 1
: (i == 7 || i == 9) ? 32
: (i == 16 || i == 18) ? 64
: (i == 29 || i == 31) ? 128
: CH[i];
if ((ks[i] != 1 && ks[i] != 3) || (ss[i] != 1 && ss[i] != 2)) {
why = "a convolution of a kernel or stride not read";
break;
}
char name[32];
snprintf(name, sizeof name, "spk.conv%d.w", i);
s->w[i] = tensor(s, name, 1,
(uint64_t)s->cout[i] * s->cin[i] * ks[i] * ks[i]);
snprintf(name, sizeof name, "spk.conv%d.b", i);
s->b[i] = tensor(s, name, 0, (uint64_t)s->cout[i]);
if (!s->w[i] || !s->b[i])
why = "a convolution's weights are missing or of another size";
}
if (!why) {
s->proj = tensor(s, "spk.proj.w", 1, 256 * 5120);
s->proj_b = tensor(s, "spk.proj.b", 0, 256);
s->mean = tensor(s, "spk.mean", 0, 256);
if (!s->proj || !s->proj_b || !s->mean)
why = "the projection is missing";
}
if (!why) {
int n = n_threads;
struct janas_cpu_layout lay;
if (n <= 0)
n = janas_cpu_layout(NULL, 0, &lay) == 0 ? lay.n_p_cores : 1;
s->pool = janas_pool_create(n > 0 ? n : 1, NULL);
if (!s->pool)
why = "cannot start the threads";
}
if (why) {
snprintf(err, err_len, "%s: %s", path, why);
janas_speaker_close(s);
return NULL;
}
return s;
}
/* ---- the filter banks ---- */
static double mel(double f)
{
return 1127.0 * log(1.0 + f / 700.0);
}
float *janas_speaker_fbank(const float *pcm, size_t n, uint32_t *frames)
{
const int len = 400, shift = 160, nfft = 512, bins = 256;
*frames = 0;
if (n < (size_t)len)
return calloc(1, sizeof(float));
size_t nf = 1 + (n - (size_t)len) / (size_t)shift;
if (nf > UINT32_MAX)
return NULL;
float *out = malloc(sizeof(float) * nf * MELS);
struct afft *f = afft_new(nfft);
float *bank = calloc((size_t)MELS * bins, sizeof(float));
float win[400], in[1024], spec[1024], frame[400];
if (!out || !f || !bank) {
free(out);
afft_free(f);
free(bank);
return NULL;
}
for (int i = 0; i < len; i++)
win[i] = (float)(0.54 - 0.46 * cos(2 * M_PI * i / (len - 1)));
/* the triangles, on Kaldi's mel scale from 20 Hz to 8 kHz */
double lo = mel(20), hi = mel(8000), delta = (hi - lo) / (MELS + 1);
for (int m = 0; m < MELS; m++) {
double l = lo + m * delta, c = lo + (m + 1) * delta,
r = lo + (m + 2) * delta;
for (int k = 0; k < bins; k++) {
double x = mel(16000.0 / nfft * k);
double up = (x - l) / (c - l), down = (r - x) / (r - c);
double v = up < down ? up : down;
bank[m * bins + k] = v > 0 ? (float)v : 0;
}
}
for (size_t t = 0; t < nf; t++) {
const float *x = pcm + t * (size_t)shift;
float mean = 0;
for (int i = 0; i < len; i++) {
frame[i] = x[i] * 32768.0f;
mean += frame[i];
}
mean /= (float)len;
for (int i = 0; i < len; i++)
frame[i] -= mean;
for (int i = len - 1; i > 0; i--)
frame[i] -= 0.97f * frame[i - 1];
frame[0] -= 0.97f * frame[0];
memset(in, 0, sizeof in);
for (int i = 0; i < len; i++)
in[2 * i] = frame[i] * win[i]; /* (re, im) pairs */
afft_run(f, in, spec);
float pw[256];
for (int k = 0; k < bins; k++)
pw[k] =
spec[2 * k] * spec[2 * k] + spec[2 * k + 1] * spec[2 * k + 1];
for (int m = 0; m < MELS; m++) {
float e = 0;
const float *bk = bank + m * bins;
for (int k = 0; k < bins; k++)
e += bk[k] * pw[k];
out[t * MELS + m] = logf(e > 1.1920929e-07f ? e : 1.1920929e-07f);
}
}
afft_free(f);
free(bank);
*frames = (uint32_t)nf;
return out;
}
/* ---- the network ---- */
/* a convolution of in (h x w x cin, channels last) into out (h' x w' x
cout); col: the patches' room */
static void conv(struct janas_speaker *s, int i, const float *in, int h, int w,
float *out, int *oh, int *ow, float *col)
{
int k = s->kernel[i], st = s->stride[i], pad = (k - 1) / 2, cin = s->cin[i];
int H = (h + 2 * pad - k) / st + 1, Wd = (w + 2 * pad - k) / st + 1;
size_t cols = (size_t)k * k * cin;
for (int y = 0; y < H; y++)
for (int x = 0; x < Wd; x++) {
float *c = col + ((size_t)y * Wd + x) * cols;
for (int ky = 0; ky < k; ky++)
for (int kx = 0; kx < k; kx++) {
int sy = y * st + ky - pad, sx = x * st + kx - pad;
float *d = c + ((size_t)ky * k + kx) * cin;
if (sy < 0 || sy >= h || sx < 0 || sx >= w)
memset(d, 0, sizeof(float) * (size_t)cin);
else
memcpy(d, in + ((size_t)sy * w + sx) * cin,
sizeof(float) * (size_t)cin);
}
}
vk_matmul_f16(s->pool, s->w[i], s->b[i], col, out, (uint32_t)s->cout[i],
(uint32_t)cols, (uint32_t)(H * Wd), 0);
*oh = H;
*ow = Wd;
}
static void relu(float *x, size_t n)
{
for (size_t i = 0; i < n; i++)
x[i] = x[i] > 0 ? x[i] : 0;
}
int janas_speaker_embed(struct janas_speaker *s, const float *fbank,
uint32_t f0, uint32_t nf, float *emb)
{
if (nf < 8)
return -1;
/* the input, frequency down, time across: h 80 x w nf x 1 */
size_t big = (size_t)MELS * nf * 32;
float *a = malloc(sizeof(float) * big), *b = malloc(sizeof(float) * big);
float *c = malloc(sizeof(float) * big), *t = malloc(sizeof(float) * big);
float *col = malloc(sizeof(float) * (size_t)MELS * nf * 9 * 32);
int rc = -1;
if (!a || !b || !c || !t || !col)
goto end;
for (int m = 0; m < MELS; m++) {
double mean = 0;
for (uint32_t i = 0; i < nf; i++)
mean += fbank[(size_t)(f0 + i) * MELS + m];
mean /= nf;
for (uint32_t i = 0; i < nf; i++)
a[(size_t)m * nf + i] =
(float)(fbank[(size_t)(f0 + i) * MELS + m] - mean);
}
int h = MELS, w = (int)nf, oh, ow;
conv(s, 0, a, h, w, b, &oh, &ow, col);
relu(b, (size_t)oh * ow * 32);
h = oh;
w = ow;
/* the residual blocks: x in b */
static const int BLOCKS[4] = {3, 4, 6, 3};
int ci = 1;
float *x = b, *y = a, *z = c;
for (int l = 0; l < 4; l++)
for (int k = 0; k < BLOCKS[l]; k++) {
int down = l > 0 && k == 0;
int ia = ci, ib = ci + 1, is = down ? ci + 2 : -1;
ci += down ? 3 : 2;
int h1, w1, h2, w2;
conv(s, ia, x, h, w, y, &h1, &w1, col);
relu(y, (size_t)h1 * w1 * s->cout[ia]);
conv(s, ib, y, h1, w1, z, &h2, &w2, col);
const float *sc = x;
if (is >= 0) {
int h3, w3;
conv(s, is, x, h, w, t, &h3, &w3, col);
sc = t;
}
size_t nn = (size_t)h2 * w2 * s->cout[ib];
for (size_t q = 0; q < nn; q++) {
float v = z[q] + sc[q];
z[q] = v > 0 ? v : 0;
}
float *tmp = x;
x = z;
z = tmp;
h = h2;
w = w2;
}
/* the statistics over time of each channel and frequency: c f order */
float st[5120];
int ch = 256;
if (h * ch != 2560) /* 80 / 8 frequencies */
goto end;
for (int cc = 0; cc < ch; cc++)
for (int f = 0; f < h; f++) {
double sum = 0;
for (int q = 0; q < w; q++)
sum += x[((size_t)f * w + q) * ch + cc];
double mean = sum / w, var = 0;
for (int q = 0; q < w; q++) {
double d = x[((size_t)f * w + q) * ch + cc] - mean;
var += d * d;
}
var = w > 1 ? var / (w - 1) : 0;
st[cc * h + f] = (float)mean;
st[2560 + cc * h + f] = (float)sqrt(var + 1e-8);
}
vk_matmul_f16(s->pool, s->proj, s->proj_b, st, emb, DIM, 5120, 1, 0);
for (int i = 0; i < DIM; i++)
emb[i] -= s->mean[i];
rc = 0;
end:
free(a);
free(b);
free(c);
free(t);
free(col);
return rc;
}