37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
/*
* llm_gen.c - greedy generation with and without speculative decoding.
*
* Generates the same continuation twice, first without speculation and then
* with lookup drafts, checks that the tokens are identical, and reports the
* speed of both and the acceptance of the drafts.
*
* Usage: llm_gen <model.jns> <prompt.int32> <n_prompt> <n_new> <k> [auto]
* [mtp=<mtp.jns>] [conf=<min probability of an MTP draft>]
* With mtp= the drafts come from the model's MTP block instead of lookup.
* With cpus=<list> (e.g. 0-11 or 18,19) the compute threads run on those
* CPUs, one each (to run it beside another process on other cores).
* LLM_GEN_DUMP=<path> writes the first run's tokens there (int32).
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "common/sysinfo.h"
#include "llm/generate.h"
#include "llm/model.h"
/* What the planner has learned: per tenth of guessed survival chance, how
many drafts it tried and how many survived. */
static void report_calibration(const struct janas_gen_stats *s)
{
int any = 0;
for (int b = 0; b < 10; b++)
any |= s->cal_tried[b] != 0;
if (!any)
return;
printf("calibration, guessed chance -> tried, survived:");
for (int b = 0; b < 10; b++)
if (s->cal_tried[b])
printf(" %.1f-%.1f: %u/%u (%.0f%%)", b / 10.0, (b + 1) / 10.0,
s->cal_won[b], s->cal_tried[b],
100.0 * s->cal_won[b] / s->cal_tried[b]);
printf("\n");
}
static void report(const char *name, const struct janas_gen_stats *s)
{
printf("%-8s prefill %u tokens %.2f s (%.1f token/s); decode %u tokens "
"%.2f s (%.1f token/s), %u passes",
name, s->prompt_tokens, s->prefill_seconds,
s->prompt_tokens / s->prefill_seconds, s->new_tokens,
s->decode_seconds, s->new_tokens / s->decode_seconds, s->passes);
if (s->new_tokens)
printf(", %u of %u forced (%.0f%%)", s->forced_tokens, s->new_tokens,
100.0 * s->forced_tokens / s->new_tokens);
if (s->sample_seconds > 0)
printf(", sampling %.2f s (%.0f%%)", s->sample_seconds,
100 * s->sample_seconds / s->decode_seconds);
if (s->draft_seconds > 0)
printf(", drafting %.2f s (%.0f%% of decode)", s->draft_seconds,
100 * s->draft_seconds / s->decode_seconds);
if (s->spec_passes)
printf(", %u verifying: %u of %u drafted accepted (%.0f%%), %.2f "
"tokens per pass",
s->spec_passes, s->accepted, s->drafted,
100.0 * s->accepted / s->drafted,
(double)s->new_tokens / s->passes);
if (s->auto_off_passes)
printf(", %u passes with speculation paused", s->auto_off_passes);
printf("\n");
}
/* By position in the chain: how often the draft there was accepted, of all
verified there and of those whose predecessor was accepted; and by the
tokens in a pass, its mean time against a pass of one in the run without
drafts (plain). */
static void report_chain(const struct janas_gen_stats *s,
const struct janas_gen_stats *plain)
{
if (!s->pos_tried[0])
return;
printf("chain, position: accepted/verified (%%, %% given the one "
"before):");
for (int i = 0; i < JANAS_GEN_MAX_K && s->pos_tried[i]; i++) {
uint32_t before = i ? s->pos_won[i - 1] : s->pos_tried[0];
printf(" %d: %u/%u (%.0f%%, %.0f%%)", i + 1, s->pos_won[i],
s->pos_tried[i], 100.0 * s->pos_won[i] / s->pos_tried[i],
before ? 100.0 * s->pos_won[i] / before : 0.0);
}
printf("\n");
double one = plain->pass_count[1]
? plain->pass_seconds[1] / plain->pass_count[1]
: 0;
printf("pass, tokens: passes, ms (x a pass of one):");
for (int n = 1; n < JANAS_GEN_MAX_K + 2; n++)
if (s->pass_count[n]) {
double t = s->pass_seconds[n] / s->pass_count[n];
printf(" %d: %u, %.1f (%.2f)", n, s->pass_count[n], t * 1e3,
one > 0 ? t / one : 0.0);
}
printf("\n");
}
int main(int argc, char **argv)
{
if (argc < 6) {
fprintf(stderr,
"usage: llm_gen <model.jns> <prompt.int32> <n_prompt> "
"<n_new> <k> [auto] [mtp=<mtp.jns>] [conf=<p>] "
"[threads=<n>] [cache=<MiB>] [ctx=<n>] [temp=<t>] [seed=<n>] "
"[draft=<small.jns>]\n");
return 2;
}
int32_t prompt[8192];
FILE *f = fopen(argv[2], "rb");
if (!f)
return 1;
size_t avail = fread(prompt, sizeof(int32_t), 8192, f);
fclose(f);
uint32_t np = (uint32_t)atoi(argv[3]), nn = (uint32_t)atoi(argv[4]);
if (np == 0 || np > avail || nn == 0)
return 2;
int use_auto = 0;
const char *mtp_path = NULL;
float min_conf = 0.0f;
/* the product picks its own thread count; the default here is the one
this test has always used, so old numbers stay comparable */
int n_compute = 12, n_ctx = 4096;
uint64_t cache_mib = 20480;
float temp = 0.0f; /* greedy, so that the two runs can be compared */
const char *draft_path = NULL;
uint64_t seed = 20260922;
float top_p = 0.8f;
int cpu_list[64], n_list = 0;
for (int i = 6; i < argc; i++) {
if (strcmp(argv[i], "auto") == 0)
use_auto = 1;
else if (strncmp(argv[i], "mtp=", 4) == 0)
mtp_path = argv[i] + 4;
else if (strncmp(argv[i], "conf=", 5) == 0)
min_conf = (float)atof(argv[i] + 5);
else if (strncmp(argv[i], "threads=", 8) == 0)
n_compute = atoi(argv[i] + 8);
else if (strncmp(argv[i], "cache=", 6) == 0)
cache_mib = (uint64_t)atol(argv[i] + 6);
else if (strncmp(argv[i], "ctx=", 4) == 0)
n_ctx = atoi(argv[i] + 4);
else if (strncmp(argv[i], "temp=", 5) == 0)
temp = (float)atof(argv[i] + 5);
else if (strncmp(argv[i], "draft=", 6) == 0)
draft_path = argv[i] + 6;
else if (strncmp(argv[i], "seed=", 5) == 0)
seed = strtoull(argv[i] + 5, NULL, 10);
else if (strncmp(argv[i], "cpus=", 5) == 0) {
/* a, a-b, comma separated */
for (const char *c = argv[i] + 5; *c && n_list < 64;) {
char *e;
long a = strtol(c, &e, 10), b = a;
if (e == c)
break;
if (*e == '-')
b = strtol(e + 1, &e, 10);
for (long k = a; k <= b && n_list < 64; k++)
cpu_list[n_list++] = (int)k;
c = *e == ',' ? e + 1 : e;
}
} else if (strncmp(argv[i], "top_p=", 6) == 0)
top_p = (float)atof(argv[i] + 6);
else if (strncmp(argv[i], "seed=", 5) == 0)
seed = (uint64_t)strtoull(argv[i] + 5, NULL, 10);
}
int icpus[8] = {12, 13, 14, 15, 16, 17, 18, 19};
struct janas_cpu_layout lay;
int ccpus[12] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11};
const int *compute = ccpus;
if (n_list > 0) {
compute = cpu_list;
n_compute = n_list;
} else if (n_compute != 12 && janas_cpu_layout(NULL, 0, &lay) == 0) {
if (n_compute < 1 || n_compute > lay.n)
n_compute = lay.n;
compute = lay.cpus;
}
struct janas_llm_options mo = {.cache_bytes = cache_mib << 20,
.n_ctx = (uint32_t)n_ctx,
.compute_cpus = compute,
.n_compute = n_compute,
.io_cpus = icpus,
.n_io = 8,
.mtp_path = mtp_path};
char err[256];
struct janas_llm_model *m =
janas_llm_model_load(argv[1], &mo, err, sizeof(err));
if (!m) {
fprintf(stderr, "%s: %s\n", argv[1], err);
return 1;
}
/* the drafting model shares the target's pool: two of them would take
the cores from each other */
struct janas_llm_model *draft = NULL;
if (draft_path) {
struct janas_llm_options dmo = mo;
dmo.mtp_path = NULL;
dmo.cache_bytes = (uint64_t)2048 << 20;
dmo.shared_compute = janas_llm_model_compute(m);
draft = janas_llm_model_load(draft_path, &dmo, err, sizeof(err));
if (!draft) {
fprintf(stderr, "%s: %s\n", draft_path, err);
return 1;
}
}
int32_t *a = malloc(nn * sizeof(int32_t)),
*b = malloc(nn * sizeof(int32_t));
/* the same seed on both runs: at a temperature above zero the two
still draw the same tokens unless the drafts change the outcome */
struct janas_gen_options off = {.spec_mode = JANAS_SPEC_OFF,
.spec_k = 1,
.eos = -1,
.temperature = temp,
/* the product's own sampling */
.top_k = temp > 0 ? 20 : 0,
.top_p = temp > 0 ? top_p : 0.0f,
.seed = seed};
int has_mtp = janas_llm_model_has_mtp(m); /* a file, or in the model */
struct janas_gen_options on = {.spec_mode = draft ? JANAS_SPEC_DRAFT
: has_mtp ? JANAS_SPEC_MTP
: JANAS_SPEC_LOOKUP,
.draft = draft,
.spec_k = atoi(argv[5]),
.spec_auto = use_auto,
.eos = -1,
.spec_min_conf = min_conf,
.temperature = temp,
.top_k = temp > 0 ? 20 : 0,
.top_p = temp > 0 ? top_p : 0.0f,
.seed = seed};
struct janas_gen_stats sa, sb;
/* warm-up run so both measured runs see the same expert cache */
int na = janas_llm_generate(m, prompt, np, nn, &off, a, &sa);
na = janas_llm_generate(m, prompt, np, nn, &off, a, &sa);
int nb = janas_llm_generate(m, prompt, np, nn, &on, b, &sb);
if (na < 0 || nb < 0) {
fprintf(stderr, "generation failed\n");
return 1;
}
/* LLM_GEN_DUMP=<path>: the first run's tokens there (int32), to read
or compare against another run's */
const char *dump = getenv("LLM_GEN_DUMP");
if (dump) {
FILE *f = fopen(dump, "wb");
if (!f || fwrite(a, sizeof(int32_t), (size_t)na, f) != (size_t)na)
fprintf(stderr, "%s: not written\n", dump);
if (f)
fclose(f);
}
printf("first tokens:");
for (int i = 0; i < 12 && i < na; i++)
printf(" %d", a[i]);
printf("\n");
printf("first tokens:");
for (int i = 0; i < 12 && i < nb; i++)
printf(" %d", b[i]);
printf("\n");
report("normal", &sa);
report(draft ? "draft"
: has_mtp ? (on.spec_auto ? "mtp+a" : "mtp")
: (on.spec_auto ? "lookup+a" : "lookup"),
&sb);
report_calibration(&sb);
report_chain(&sb, &sa);
int same = na == nb && memcmp(a, b, (size_t)na * sizeof(int32_t)) == 0;
/* where they part: on the CPU the drafts never change the text, on
the GPU its sums in another order may (its quality is measured
against a reference instead, llm_eval) */
int first = 0;
while (first < na && first < nb && a[first] == b[first])
first++;
printf("tokens identical: %s (%d and %d", same ? "yes" : "NO", na, nb);
if (!same)
printf(", the first difference at token %d", first);
printf("); speed-up %.2fx\n", (sb.new_tokens / sb.decode_seconds) /
(sa.new_tokens / sa.decode_seconds));
free(a);
free(b);
char tune[512];
janas_llm_model_tuning(m, tune, sizeof(tune));
printf("tuning: %s\n", tune);
janas_llm_model_free(m);
return same ? 0 : 1;
}