37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
/*
* api_reply.c - the public API: the reply, piece by piece, and its counters
* (see include/janas/llm.h).
*/
#include <stdlib.h>
#include <string.h>
#include <sys/resource.h>
#include <time.h>
#include "llm/chat.h"
#include "llm/metrics.h"
#include "llm/model.h"
int janas_api_is_stop(const janas_llm *llm, int32_t t)
{
for (int i = 0; i < llm->n_stop; i++)
if (t == llm->stop[i])
return 1;
return 0;
}
/* Bytes of pend that form whole UTF-8 characters, at most max. */
static size_t whole_chars(const char *s, size_t n, size_t max)
{
size_t i = 0;
while (i < n) {
unsigned char b = (unsigned char)s[i];
size_t k = b < 0x80 ? 1
: (b >> 5) == 6 ? 2
: (b >> 4) == 14 ? 3
: (b >> 3) == 30 ? 4
: 1;
if (i + k > n || i + k > max)
break;
i += k;
}
return i;
}
static double cpu_now(void)
{
struct timespec t;
clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &t);
return (double)t.tv_sec + 1e-9 * (double)t.tv_nsec;
}
/*
* The time left to read the prompt: a token's cost taken as a line in its
* position, a + b p (attention grows with the context it looks at), fitted
* on the blocks read so far and integrated over the positions left. On a
* 260,954-token prompt of Qwen3-Next-80B the elapsed time followed a n +
* b n^2 to R^2 0.99999 (issue #9), where the latest speed promised 2.5
* times too little at 32K. Never below what the last block's speed says:
* the cost does not fall as the context grows. A line through the first
* few blocks tilts with their noise (an ETA many times too long at 1,024
* tokens, issue #9), so the estimate moves from the last block's speed to
* the fitted line over the first ETA_BLOCKS blocks fitted.
*/
#define ETA_BLOCKS 16
static double input_eta(const janas_llm_chat *c, uint32_t left)
{
if (left == 0)
return 0;
if (c->eta_last < 0)
return -1;
double flat = c->eta_last * left;
double w = c->eta_w, det = w * c->eta_sxx - c->eta_sx * c->eta_sx;
if (c->eta_steps < 4 || det <= 0)
return flat;
double b = (w * c->eta_sxc - c->eta_sx * c->eta_sc) / det;
double a = (c->eta_sc - b * c->eta_sx) / w;
double p0 = janas_llm_session_computed(c->s), p1 = p0 + left;
double fit = a * (p1 - p0) + 0.5 * b * (p1 * p1 - p0 * p0);
if (fit <= flat)
return flat;
double k = (double)(c->eta_steps - 2) / ETA_BLOCKS;
return flat + (k < 1 ? k : 1) * (fit - flat);
}
/* One block of the prompt read: n tokens from position p0, in dt seconds.
The first two blocks are left out: they warm the caches and the tuner. */
static void eta_note(janas_llm_chat *c, uint32_t p0, uint32_t n, double dt)
{
if (n == 0 || dt <= 0)
return;
double cost = dt / n, x = p0 + 0.5 * n;
c->eta_last = cost;
if (++c->eta_steps <= 2)
return;
c->eta_w += n;
c->eta_sx += n * x;
c->eta_sc += n * cost;
c->eta_sxx += n * x * x;
c->eta_sxc += n * x * cost;
}
void janas_api_update_stats(janas_llm_chat *c)
{
struct janas_gen_stats st;
janas_llm_session_stats(c->s, &st);
struct janas_llm_chat_stats *s = &c->stats;
s->size = sizeof(*s);
s->output_tokens = c->reply_tokens;
s->context_used = janas_llm_session_length(c->s);
s->context_size = c->llm->n_ctx;
s->drafted = st.drafted - c->st0.drafted;
s->accepted = st.accepted - c->st0.accepted;
s->auto_off_passes = st.auto_off_passes - c->st0.auto_off_passes;
s->input_seconds = st.prefill_seconds - c->st0.prefill_seconds;
s->output_seconds = st.decode_seconds - c->st0.decode_seconds;
s->total_seconds = janas_api_now() - c->t_send;
struct janas_expert_cache_stats cs;
janas_expert_cache_stats(janas_llm_model_cache(c->llm->m), &cs);
s->experts_used = cs.requests - c->cs0.requests;
/* the experts read ahead count as read: they came from the file, and
left out they made the cache look warmer than it is and the disk
idler (1 Oct 2026: 108 GB "read" against 161 without them) */
s->experts_read =
cs.misses - c->cs0.misses + cs.prefetch_used - c->cs0.prefetch_used;
s->bytes_read = cs.bytes_read - c->cs0.bytes_read + cs.prefetch_bytes -
c->cs0.prefetch_bytes;
s->io_wait_seconds = cs.wait_seconds - c->cs0.wait_seconds;
/* the reading of the prompt, and the rest of what a reply costs */
uint32_t done = janas_llm_session_computed(c->s) - c->kv_from;
if (janas_llm_session_computed(c->s) < c->kv_from)
done = 0;
s->input_done = s->stage == JANAS_LLM_STAGE_INPUT && done < s->input_tokens
? done
: s->stage == JANAS_LLM_STAGE_IDLE ? 0
: s->input_tokens;
s->first_token_seconds = c->t_first;
s->input_cpu_seconds = c->input_cpu;
s->input_threads = janas_llm_model_threads(c->llm->m, 2);
s->output_threads = janas_llm_model_threads(c->llm->m, 0);
s->context_bytes =
c->llm->kv_token * s->context_used +
c->llm->kv_ring_token * (s->context_used < c->llm->kv_ring
? s->context_used
: c->llm->kv_ring);
struct rusage ru;
if (getrusage(RUSAGE_SELF, &ru) == 0)
s->peak_rss_bytes = (uint64_t)ru.ru_maxrss * 1024;
s->input_eta_seconds = s->stage == JANAS_LLM_STAGE_INPUT
? input_eta(c, s->input_tokens - s->input_done)
: 0;
}
/*
* A block of the prompt read: 1 when more is left, 0 when it has all been
* read (the prompt is kept then, where it is to be), -1 on an error.
*/
static int read_input(janas_llm_chat *c)
{
double c0 = cpu_now(), t0 = janas_api_now();
uint32_t p0 = janas_llm_session_computed(c->s);
int r = janas_llm_session_prefill_step(c->s);
c->input_cpu += cpu_now() - c0;
uint32_t p1 = janas_llm_session_computed(c->s);
if (r >= 0 && p1 > p0)
eta_note(c, p0, p1 - p0, janas_api_now() - t0);
if (r == 0 && c->keep_prompt) {
c->keep_prompt = 0;
janas_api_keep_prompt(c); /* read already: only kept */
}
return r;
}
/*
* Returns up to cap bytes of whole characters from pend. Once the reply has
* ended, a character the last token left unfinished (max_reply can stop a
* reply between the tokens of an emoji) will never be finished: its bytes
* are dropped, since what comes out is promised to be whole characters, and
* a client that decodes it would fail on them.
*/
static size_t emit(janas_llm_chat *c, char *buf, size_t cap)
{
size_t w = whole_chars(c->pend, c->npend, cap);
memcpy(buf, c->pend, w);
memmove(c->pend, c->pend + w, c->npend - w);
c->npend -= w;
if (c->ended && w == 0 && c->npend < 4)
c->npend = 0; /* a character left half written */
return w;
}
/* Bytes for the caller, as many as pend has room for. */
static void give(janas_llm_chat *c, const char *s, size_t n)
{
if (n > sizeof(c->pend) - c->npend)
n = sizeof(c->pend) - c->npend;
memcpy(c->pend + c->npend, s, n);
c->npend += n;
}
/*
* A token of the reply, where it belongs. The reasoning markers are not
* part of the text: the opening one is written into the prompt and never
* generated, so letting the closing one through printed half a pair; and
* where a model writes both, the colour of the reasoning says the same
* thing twice. Either both or neither: neither. A tool call, from its
* opening token to its closing one, is not text either: it is read into a
* call of the reply.
*/
static void take(janas_llm_chat *c, int32_t t)
{
const janas_llm *llm = c->llm;
char tmp[sizeof(c->pend)];
if (!c->in_think && t != llm->think_open && t != llm->think_close)
c->answer_tokens++; /* text or a call: what max_answer counts */
if (t == llm->think_open || t == llm->think_close) {
c->in_think = t == llm->think_open;
/* Gemma names its channel before the reasoning: "thought\n" */
c->chan_label = c->in_think && llm->format == FORMAT_GEMMA;
janas_api_record(c, t, JANAS_LLM_PART_MARK, "", 0);
return;
}
if (c->chan_label) {
char lab[32];
size_t ln = janas_tokenizer_decode(llm->tok, t, lab, sizeof(lab));
if (ln < sizeof(lab) && ((ln == 7 && !memcmp(lab, "thought", 7)) ||
(ln == 1 && *lab == '\n'))) {
c->chan_label = lab[0] != '\n';
janas_api_record(c, t, JANAS_LLM_PART_MARK, "", 0);
return;
}
c->chan_label = 0;
}
if (c->tools && t == llm->call_open && !c->in_think && !c->in_call) {
c->in_call = 1;
c->body.n = 0;
janas_api_record(c, t, JANAS_LLM_PART_MARK, "", 0);
return;
}
if (c->in_call && t == llm->call_close) {
janas_api_record(c, t, JANAS_LLM_PART_MARK, "", 0);
struct janas_buf text = {0};
janas_api_close_call(c, &text);
if (text.n) {
janas_api_rtext_add(c, text.p, text.n);
give(c, text.p, text.n);
}
janas_buf_free(&text);
return;
}
size_t n = janas_tokenizer_decode(llm->tok, t, tmp, sizeof(tmp));
if (n > sizeof(tmp))
n = sizeof(tmp);
if (c->in_call) {
janas_buf_put(&c->body, tmp, n);
janas_api_record(c, t, JANAS_LLM_PART_CALL, tmp, n);
return;
}
if (c->in_think)
c->stats.reasoning_tokens++; /* what max_reasoning counts */
janas_api_record(
c, t, c->in_think ? JANAS_LLM_PART_REASONING : JANAS_LLM_PART_ANSWER,
tmp, n);
if (!c->in_think)
janas_api_rtext_add(c, tmp, n); /* the answer, for memo_store */
give(c, tmp, n);
}
/*
* The reasoning at max_reasoning tokens, closed for the model: a sentence of
* ours, the format's closing mark (</think>, Gemma's <channel|>) and, after
* </think>, the blank line the model itself writes there; the answer follows
* (the drafts beyond are dropped). The sentence is what makes a small model
* answer: cut at 32 tokens with janas-chat's system message, Qwen3.5-0.8B and
* 2B given the mark alone went on reasoning in their answer and mostly looped
* to the end of the reply (20 of 36 replies ended by the model, the three
* models with Qwen3-4B), given the sentence first answered (35 of 36, the
* replies a third as long: 96 tokens on average against 265); Qwen3-4B answered
* either way (13 Oct 2026). The sentence is reasoning, shown as such and
* counted: the reasoning ends that many tokens past the ceiling. Without a
* ceiling a model whose reasoning never closes - issue #21's, its arithmetic
* broken - writes on to the end of the context, a chat's max_tokens counting
* the answer alone.
*/
static void close_reasoning(janas_llm_chat *c)
{
const janas_llm *llm = c->llm;
if (llm->think_close < 0)
return; /* no mark: the context ends the reply */
static const char say[] =
"\n\nI have thought enough; now I give the answer.\n";
int32_t ids[80];
long n = janas_tokenizer_encode(llm->tok, say, sizeof(say) - 1, 0, ids, 64);
if (n < 0 || n > 64)
n = 0;
ids[n++] = llm->think_close;
if (llm->format != FORMAT_GEMMA) {
long m = janas_tokenizer_encode(llm->tok, "\n\n", 2, 0, ids + n, 8);
if (m > 0 && m <= 8)
n += m;
}
if (janas_llm_session_append(c->s, ids, (uint32_t)n) != 0)
return; /* no room for it: the context ends the reply */
c->stats.reasoning_cut = 1;
/* taken one a call (janas_llm_chat_next): the reasoning's last piece
is returned as reasoning before the answer begins */
memcpy(c->forced, ids, (size_t)n * sizeof(*ids));
c->n_forced = (uint32_t)n;
c->i_forced = 0;
}
int32_t janas_llm_chat_next(janas_llm_chat *c, char *buf, int32_t cap,
int32_t *len)
{
if (!c || !buf || !len || cap < 8)
return janas_api_fail(JANAS_LLM_EINVAL, "invalid argument (cap >= 8)");
*len = 0;
if (!c->replying)
return JANAS_LLM_DONE;
/* the prompt, a block at a time: the caller hears between blocks */
if (c->stats.stage == JANAS_LLM_STAGE_INPUT) {
int r = read_input(c);
if (r > 0) {
janas_api_update_stats(c);
janas_metrics_progress(c, 0);
return JANAS_LLM_OK;
}
c->stats.stage = JANAS_LLM_STAGE_OUTPUT;
if (r < 0) {
c->replying = 0;
c->stats.finish = JANAS_LLM_FINISH_ERROR;
c->stats.stage = JANAS_LLM_STAGE_DONE;
janas_api_update_stats(c);
janas_metrics_reply(c);
return janas_api_fail(JANAS_LLM_EFAIL, "generation failed");
}
janas_api_update_stats(c);
if (c->stats.input_tokens >= 1024)
janas_metrics_progress(c, 1); /* the whole prompt, read */
}
if (!c->ended && c->npend < 4) {
int32_t t = -1;
int r = 0, got = 0;
if ((c->p.max_reply > 0 &&
c->reply_tokens >= (uint32_t)c->p.max_reply) ||
(c->p.max_answer > 0 &&
c->answer_tokens >= (uint32_t)c->p.max_answer)) {
c->ended = 1;
c->stats.finish = JANAS_LLM_FINISH_LENGTH;
} else if (c->i_forced < c->n_forced) {
t = c->forced[c->i_forced++]; /* in the session already */
got = 1;
} else {
r = janas_llm_session_next(c->s, &t);
got = r == 0;
}
if (got && c->t_first == 0) { /* the first token of the reply */
c->t_first = janas_api_now() - c->t_send;
if (c->t_first <= 0)
c->t_first = 1e-6;
c->stats.first_token_seconds = c->t_first;
janas_metrics_first_token(c);
}
if (r != 0) {
c->ended = 1;
c->stats.finish =
r > 0 ? JANAS_LLM_FINISH_CONTEXT : JANAS_LLM_FINISH_ERROR;
} else if (got && janas_api_is_stop(c->llm, t)) {
c->ended = c->closed = 1;
c->stats.finish =
c->n_calls ? JANAS_LLM_FINISH_TOOLS : JANAS_LLM_FINISH_STOP;
} else if (got) {
c->reply_tokens++;
take(c, t);
if (c->in_think && c->p.max_reasoning > 0 &&
c->stats.reasoning_tokens >= (uint32_t)c->p.max_reasoning &&
c->i_forced >= c->n_forced)
close_reasoning(c);
}
janas_api_update_stats(c);
if (r != 0) {
c->replying = 0;
c->stats.stage = JANAS_LLM_STAGE_DONE;
janas_metrics_reply(c);
return r > 0 ? janas_api_fail(JANAS_LLM_EFULL,
"the context is full (%u "
"tokens)",
c->llm->n_ctx)
: janas_api_fail(JANAS_LLM_EFAIL, "generation failed");
}
}
*len = (int32_t)emit(c, buf, (size_t)cap);
if (c->ended && c->npend == 0 && *len == 0) {
c->replying = 0;
c->stats.stage = JANAS_LLM_STAGE_DONE;
janas_api_update_stats(c);
janas_metrics_reply(c);
janas_api_memo_store(c);
return JANAS_LLM_DONE;
}
return JANAS_LLM_OK;
}
int32_t janas_llm_chat_stats(const janas_llm_chat *c,
struct janas_llm_chat_stats *s)
{
if (!c || !s || s->size < sizeof(uint32_t))
return janas_api_fail(JANAS_LLM_EINVAL, "invalid argument");
uint32_t size = s->size < sizeof(c->stats) ? s->size : sizeof(c->stats);
memcpy(s, &c->stats, size);
s->size = size;
return JANAS_LLM_OK;
}