Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

35.6 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
/*
 * api_chat.c - the public API: a conversation, built message by message in
 * the model's chat format, and the reply started (see include/janas/llm.h).
 * A message is turned into tokens the way the rendered conversation would
 * be: the format's special tokens as ids, the text between them tokenized as
 * one run (so the user's text never produces special tokens).
 */
#include <stdlib.h>
#include <string.h>
#include <time.h>

#include "llm/chat.h"
#include "llm/metrics.h"

void janas_llm_chat_params_default(struct janas_llm_chat_params *p)
{
    memset(p, 0, sizeof(*p));
    p->size = sizeof(*p);
    p->temperature = 0.7f;
    p->top_k = 20;
    p->top_p = 0.8f;
    p->speculate = 1;
    p->thinking = -1;
    p->recap = 1;
}

static int32_t gen_options(const janas_llm_chat *c, struct janas_gen_options *o)
{
    const struct janas_llm_chat_params *p = &c->p;
    if (!(p->temperature >= 0) || p->top_k < 0 || !(p->top_p >= 0) ||
        !(p->min_p >= 0) || p->max_reply < 0 || p->max_reasoning < 0 ||
        p->top_logprobs < 0 || p->top_logprobs > JANAS_SAMPLE_MAX_TOP ||
        !(p->presence_penalty >= -2 && p->presence_penalty <= 2) ||
        !(p->frequency_penalty >= -2 && p->frequency_penalty <= 2))
        return janas_api_fail(JANAS_LLM_EINVAL, "invalid chat parameters");
    memset(o, 0, sizeof(*o));
    int mtp = c->llm->has_mtp;
    o->spec_mode = !p->speculate   ? JANAS_SPEC_OFF
                   : c->llm->draft ? JANAS_SPEC_DRAFT
                   : mtp           ? JANAS_SPEC_MTP
                                   : JANAS_SPEC_LOOKUP;
    o->draft = c->llm->draft;
    /*
     * The ceiling on how many tokens may be proposed, not how many will be:
     * a planner picks the number from the cost of a longer pass and the rate
     * at which the drafts in each position have been surviving. Until 22 Sep
     * 2026 the drafts copied from the context had no planner and always
     * asked for four, which on ordinary prose was slower than not drafting
     * at all - 0.89x on Qwen3-4B, 0.82x on Qwen3-30B - because acceptance
     * falls to about one in nine and a failed verification costs a pass over
     * five tokens.
     */
    o->spec_k = mtp ? 8 : 4;
    o->spec_auto = 1;
    o->eos = -1;
    o->temperature = p->temperature;
    o->top_k = p->top_k;
    o->top_p = p->top_p;
    o->min_p = p->min_p;
    o->presence_penalty = p->presence_penalty;
    o->frequency_penalty = p->frequency_penalty;
    o->logprobs = p->logprobs != 0;
    o->top_logprobs = p->top_logprobs;
    o->seed = p->seed ? p->seed
                      : (uint64_t)time(NULL) * 0x9e3779b97f4a7c15ull ^
                            (uint64_t)(uintptr_t)c;
    memcpy(o->stop, c->llm->stop, sizeof(o->stop));
    o->n_stop = c->llm->n_stop;
    return JANAS_LLM_OK;
}

int32_t janas_api_options(const janas_llm_chat *c, struct janas_gen_options *o)
{
    return gen_options(c, o);
}

static int32_t take_params(janas_llm_chat *c,
                           const struct janas_llm_chat_params *pp)
{
    struct janas_llm_chat_params p;
    janas_llm_chat_params_default(&p);
    if (pp) {
        if (pp->size < sizeof(uint32_t) || pp->size > sizeof(p))
            return janas_api_fail(JANAS_LLM_EINVAL, "parameters: unknown size");
        memcpy(&p, pp, pp->size);
        p.size = sizeof(p);
    }
    struct janas_llm_chat_params old = c->p;
    c->p = p;
    struct janas_gen_options o;
    int32_t rc = gen_options(c, &o);
    if (rc == 0 && c->s && janas_llm_session_set_options(c->s, &o) != 0)
        rc = janas_api_fail(JANAS_LLM_EINVAL, "invalid chat parameters");
    if (rc != 0)
        c->p = old;
    return rc;
}

int32_t janas_llm_chat_create(janas_llm *llm,
                              const struct janas_llm_chat_params *p,
                              janas_llm_chat **out)
{
    if (!llm || !out)
        return janas_api_fail(JANAS_LLM_EINVAL, "no model or no output handle");
    *out = NULL;
    if (llm->chat)
        return janas_api_fail(JANAS_LLM_EBUSY, "the model already has a chat");
    janas_llm_chat *c = calloc(1, sizeof(*c));
    if (!c)
        return janas_api_fail(JANAS_LLM_ENOMEM, "out of memory");
    c->llm = llm;
    int32_t rc = take_params(c, p);
    struct janas_gen_options o;
    if (rc == 0 && (rc = gen_options(c, &o)) == 0 &&
        !(c->s = janas_llm_session_create(llm->m, &o)))
        rc = janas_api_fail(JANAS_LLM_ENOMEM, "out of memory");
    if (rc != 0) {
        free(c);
        return rc;
    }
    llm->chat = c;
    *out = c;
    return JANAS_LLM_OK;
}

void janas_llm_chat_destroy(janas_llm_chat *c)
{
    if (!c)
        return;
    c->llm->chat = NULL;
    /* the tools first: the session holds a filter bound to them */
    janas_api_tools_free(c);
    janas_api_msgs_free(c);
    janas_api_keep_free(c); /* before the session: it may be written */
    janas_llm_session_destroy(c->s);
    free(c->system);
    free(c->ids);
    free(c->text);
    free(c->turn);
    for (size_t i = 0; i < sizeof(c->memo) / sizeof(c->memo[0]); i++) {
        free(c->memo[i].text);
        free(c->memo[i].ids);
    }
    free(c->rtext);
    janas_api_pics_free(c->pics, c->n_pics);
    free(c->idpics);
    free(c);
}

int32_t janas_llm_chat_set_params(janas_llm_chat *c,
                                  const struct janas_llm_chat_params *p)
{
    if (!c || !p)
        return janas_api_fail(JANAS_LLM_EINVAL, "invalid argument");
    return take_params(c, p);
}

int32_t janas_llm_chat_system(janas_llm_chat *c, const char *text, int32_t len)
{
    if (!c || !text)
        return janas_api_fail(JANAS_LLM_EINVAL, "invalid argument");
    janas_api_unprepare(c);
    if (janas_llm_session_length(c->s) > 0)
        return janas_api_fail(JANAS_LLM_EINVAL,
                              "the conversation has started: reset it first");
    size_t n = janas_api_text_len(text, len);
    char *s = malloc(n + 1);
    if (!s)
        return janas_api_fail(JANAS_LLM_ENOMEM, "out of memory");
    memcpy(s, text, n);
    s[n] = 0;
    free(c->system);
    c->system = s;
    return JANAS_LLM_OK;
}

int32_t janas_llm_chat_reset(janas_llm_chat *c)
{
    if (!c)
        return janas_api_fail(JANAS_LLM_EINVAL, "no chat");
    janas_llm_session_reset(c->s);
    free(c->system);
    c->system = NULL;
    c->n_turn = 0;
    c->sys_tokens = 0;
    c->prepared = 0;
    c->tools_tokens = 0;
    c->dropped = 0;
    c->replying = c->closed = c->ended = 0;
    c->npend = 0;
    return JANAS_LLM_OK;
}

/* Message building: plain text accumulates, a special token flushes it. */
int janas_api_add_text(janas_llm_chat *c, const char *s, size_t n)
{
    if (c->n_text + n > c->cap_text) {
        size_t cap = 2 * (c->n_text + n) + 256;
        char *t = realloc(c->text, cap);
        if (!t)
            return -1;
        c->text = t;
        c->cap_text = cap;
    }
    memcpy(c->text + c->n_text, s, n);
    c->n_text += n;
    return 0;
}

int janas_api_flush_text(janas_llm_chat *c)
{
    if (c->n_text == 0)
        return 0;
    long need =
        janas_tokenizer_encode(c->llm->tok, c->text, c->n_text, 0, NULL, 0);
    if (need < 0)
        return -1;
    if (c->n_ids + (size_t)need + 1 > c->cap_ids) {
        size_t cap = 2 * (c->n_ids + (size_t)need) + 64;
        int32_t *t = realloc(c->ids, cap * sizeof(int32_t));
        if (!t)
            return -1;
        c->ids = t;
        c->cap_ids = cap;
    }
    janas_tokenizer_encode(c->llm->tok, c->text, c->n_text, 0,
                           c->ids + c->n_ids, (size_t)need);
    c->n_ids += (size_t)need;
    c->n_text = 0;
    return 0;
}

int janas_api_add_special(janas_llm_chat *c, int32_t id)
{
    if (janas_api_flush_text(c) != 0)
        return -1;
    if (c->n_ids + 1 > c->cap_ids) {
        size_t cap = 2 * c->n_ids + 64;
        int32_t *t = realloc(c->ids, cap * sizeof(int32_t));
        if (!t)
            return -1;
        c->ids = t;
        c->cap_ids = cap;
    }
    c->ids[c->n_ids++] = id;
    return 0;
}

/* The newline after a turn's end: ChatML's and Gemma's, none in Llama 3. */
int janas_api_end_turn_nl(janas_llm_chat *c)
{
    return c->llm->format == FORMAT_LLAMA3 ? 0 : janas_api_add_text(c, "\n", 1);
}

/* One message of the conversation, in the model's format. */
int janas_api_add_message(janas_llm_chat *c, const char *role, const char *text,
                          size_t n)
{
    const janas_llm *llm = c->llm;
    switch (llm->format) {
    case FORMAT_LLAMA3:
        return janas_api_add_special(c, llm->im_start) ||
               janas_api_add_text(c, role, strlen(role)) ||
               janas_api_add_special(c, llm->hdr_end) ||
               janas_api_add_text(c, "\n\n", 2) ||
               janas_api_add_text(c, text, n) ||
               janas_api_add_special(c, llm->im_end);
    case FORMAT_CHATML:
    default:
        return janas_api_add_special(c, llm->im_start) ||
               janas_api_add_text(c, role, strlen(role)) ||
               janas_api_add_text(c, "\n", 1) ||
               janas_api_add_text(c, text, n) ||
               janas_api_add_special(c, llm->im_end) ||
               janas_api_end_turn_nl(c);
    }
}

/* The start of the sequence: Gemma's <bos>, Llama 3's <|begin_of_text|>,
   nothing for ChatML. */
int janas_api_begin_sequence(janas_llm_chat *c)
{
    return c->llm->bos >= 0 ? janas_api_add_special(c, c->llm->bos) : 0;
}

int janas_api_start_reply(janas_llm_chat *c)
{
    const janas_llm *llm = c->llm;
    switch (llm->format) {
    case FORMAT_GEMMA:
        if (c->in_turn) { /* after the tools' answers: the model goes on */
            c->in_turn = 0;
            c->in_think = c->chan_label = 0;
            return janas_api_flush_text(c);
        }
        if (janas_api_add_special(c, llm->im_start) ||
            janas_api_add_text(c, "model\n", 6))
            return -1;
        /* reasoning asked for (<|think|> in the system turn): the model
           opens its channel itself; otherwise the channel is written empty
           where the template does (not E2B and E4B's) */
        c->in_think = c->chan_label = 0;
        if ((llm->thinker && c->p.thinking != 0) || !llm->empty_thought)
            return janas_api_flush_text(c);
        return janas_api_add_special(c, llm->think_open) ||
               janas_api_add_text(c, "thought\n", 8) ||
               janas_api_add_special(c, llm->think_close) ||
               janas_api_flush_text(c);
    case FORMAT_LLAMA3:
        c->in_think = 0;
        return janas_api_add_special(c, llm->im_start) ||
               janas_api_add_text(c, "assistant", 9) ||
               janas_api_add_special(c, llm->hdr_end) ||
               janas_api_add_text(c, "\n\n", 2) || janas_api_flush_text(c);
    case FORMAT_CHATML:
    default:
        if (janas_api_add_special(c, llm->im_start) ||
            janas_api_add_text(c, "assistant\n", 10))
            return -1;
        /* a reasoning model: open the reasoning, or pass it empty */
        c->in_think = 0;
        if (llm->thinker) {
            if (c->p.thinking != 0) {
                c->in_think = 1;
                return janas_api_add_special(c, llm->think_open) ||
                       janas_api_add_text(c, "\n", 1) ||
                       janas_api_flush_text(c);
            }
            return janas_api_add_special(c, llm->think_open) ||
                   janas_api_add_text(c, "\n\n", 2) ||
                   janas_api_add_special(c, llm->think_close) ||
                   janas_api_add_text(c, "\n\n", 2) || janas_api_flush_text(c);
        }
        return janas_api_flush_text(c);
    }
}

/*
 * Text the chat format writes (never the user's): the markers of the
 * reasoning and of the tools become their tokens, as they are in the
 * conversations the model was trained on. "<tool_call>" inside the system
 * message's instructions is one token there, not eleven bytes.
 */
int janas_api_add_markup(janas_llm_chat *c, const char *s, size_t n)
{
    const janas_llm *llm = c->llm;
    int gemma = llm->format == FORMAT_GEMMA;
    const struct {
        const char *text;
        int32_t id;
    } marks[] = {{"<tool_call>", llm->call_open},
                 {"</tool_call>", llm->call_close},
                 {"<tool_response>", llm->resp_open},
                 {"</tool_response>", llm->resp_close},
                 {"<think>", llm->think_open},
                 {"</think>", llm->think_close},
                 /* Gemma 4 */
                 {"<|tool_call>", gemma ? llm->call_open : -1},
                 {"<tool_call|>", gemma ? llm->call_close : -1},
                 {"<|tool_response>", gemma ? llm->resp_open : -1},
                 {"<tool_response|>", gemma ? llm->resp_close : -1},
                 {"<|\"|>", llm->quote},
                 {"<|tool>", llm->decl_open},
                 {"<tool|>", llm->decl_close}};
    size_t from = 0;
    for (size_t i = 0; i < n; i++) {
        if (s[i] != '<')
            continue;
        for (size_t k = 0; k < sizeof(marks) / sizeof(marks[0]); k++) {
            size_t m = strlen(marks[k].text);
            if (marks[k].id < 0 || n - i < m || memcmp(s + i, marks[k].text, m))
                continue;
            if (janas_api_add_text(c, s + from, i - from) ||
                janas_api_add_special(c, marks[k].id))
                return -1;
            i += m - 1;
            from = i + 1;
            break;
        }
    }
    return janas_api_add_text(c, s + from, n - from);
}

const char *janas_api_think_note(const janas_llm_chat *c)
{
    const janas_llm *llm = c->llm;
    return llm->format == FORMAT_CHATML && llm->thinker && c->p.thinking != 0 &&
                   llm->think_note[0]
               ? llm->think_note
               : NULL;
}

/* The system message, with the tools when there are: the dialect's
   template writes them into it. While the model reasons, its words on
   reasoning (janas_api_think_note) at the top, before the tools, as its
   template writes them: the message trimmed after them. */
int janas_api_add_system(janas_llm_chat *c, const char *sys, size_t n)
{
    const janas_llm *llm = c->llm;
    c->tools_tokens = 0;
    const char *note = janas_api_think_note(c);
    struct janas_buf nb = {0};
    if (note && llm->format == FORMAT_CHATML) {
        while (n && sys &&
               (*sys == ' ' || *sys == '\n' || *sys == '\t' || *sys == '\r'))
            sys++, n--;
        while (n && (sys[n - 1] == ' ' || sys[n - 1] == '\n' ||
                     sys[n - 1] == '\t' || sys[n - 1] == '\r'))
            n--;
        janas_buf_puts(&nb, note);
        if (n && (!c->tools_sys || llm->tools == JANAS_TOOLS_NONE)) {
            janas_buf_puts(&nb, "\n\n");
            janas_buf_put(&nb, sys, n);
        }
        if (nb.oom) {
            janas_buf_free(&nb);
            return -1;
        }
        if (!c->tools_sys || llm->tools == JANAS_TOOLS_NONE) {
            int err = janas_api_add_message(c, "system", nb.p, nb.n);
            janas_buf_free(&nb);
            return err;
        }
        janas_buf_puts(&nb, "\n\n"); /* before the tools */
    }
    if (llm->format == FORMAT_GEMMA) {
        /* a system turn only when there is something to say: the message,
           or <|think|> at its top when reasoning is asked for */
        int think = llm->thinker && c->p.thinking != 0;
        int tools = c->tools_sys && llm->tools == JANAS_TOOLS_GEMMA;
        if (!think && !tools && (!sys || n == 0))
            return 0;
        struct janas_buf b = {0};
        janas_tools_system(&b, tools ? JANAS_TOOLS_GEMMA : JANAS_TOOLS_NONE,
                           c->tools_sys, sys, n);
        int err = b.oom || janas_api_add_special(c, llm->im_start) ||
                  janas_api_add_text(c, "system\n", 7) ||
                  (think && (janas_api_add_special(c, llm->think_on) ||
                             janas_api_add_text(c, "\n", 1))) ||
                  janas_api_add_markup(c, b.p, b.n) ||
                  janas_api_add_special(c, llm->im_end) ||
                  janas_api_end_turn_nl(c);
        janas_buf_free(&b);
        return err ? -1 : 0;
    }
    if (!c->tools_sys || llm->tools == JANAS_TOOLS_NONE)
        return janas_api_add_message(c, "system", sys ? sys : "", n);
    /* how many of its tokens the tools take: the message without them is
       built and taken back, for the count alone */
    size_t ids0 = c->n_ids;
    if (janas_api_flush_text(c) != 0) {
        janas_buf_free(&nb);
        return -1;
    }
    ids0 = c->n_ids;
    /* the words on reasoning are the message's, not the tools' */
    size_t nn = nb.n;
    if (n)
        janas_buf_put(&nb, sys, n);
    if (nb.oom ||
        janas_api_add_message(c, "system", nb.p ? nb.p : "", nb.n) != 0 ||
        janas_api_flush_text(c) != 0) {
        janas_buf_free(&nb);
        return -1;
    }
    nb.n = nn;
    size_t plain = c->n_ids - ids0;
    c->n_ids = ids0;
    struct janas_buf b = {0};
    if (nb.n)
        janas_buf_put(&b, nb.p, nb.n);
    janas_buf_free(&nb);
    janas_tools_system(&b, llm->tools, c->tools_sys, sys, n);
    int err = b.oom || janas_api_add_special(c, llm->im_start) ||
              janas_api_add_text(c, "system\n", 7) ||
              janas_api_add_markup(c, b.p, b.n) ||
              janas_api_add_special(c, llm->im_end) ||
              janas_api_end_turn_nl(c) || janas_api_flush_text(c);
    janas_buf_free(&b);
    if (!err && c->n_ids - ids0 > plain)
        c->tools_tokens = (uint32_t)(c->n_ids - ids0 - plain);
    return err ? -1 : 0;
}

int32_t janas_llm_chat_thinking(const janas_llm_chat *c)
{
    return c && c->in_think;
}

int32_t janas_llm_reasons(const janas_llm *llm)
{
    return llm ? llm->thinker : 0;
}

/*
 * Before the oldest exchanges go, the model is asked for a short recap of
 * the conversation. It is one more turn, so it stays in the window when the
 * older ones are dropped, and the thread of the conversation with it. A few
 * seconds, once, when the context fills.
 */

static const char *const RECAP_ASK =
    "Before we go on: sum up our conversation so far in about 100 words, "
    "keeping the names, the facts and anything you were asked to remember. "
    "Write only the summary.";

static int recap(janas_llm_chat *c)
{
    int32_t *keep = NULL;
    size_t n_keep = c->n_ids;
    if (n_keep) { /* the user's message is already built: set it aside */
        keep = malloc(n_keep * sizeof(int32_t));
        if (!keep)
            return -1;
        memcpy(keep, c->ids, n_keep * sizeof(int32_t));
    }
    c->n_ids = c->n_text = 0;
    c->n_idpics = 0;
    int err = 0;
    if (!c->closed)
        err |= janas_api_add_special(c, c->llm->im_end);
    err |= janas_api_end_turn_nl(c);
    err |= janas_api_add_message(c, "user", RECAP_ASK, strlen(RECAP_ASK));
    err |= janas_api_start_reply(c);
    if (!err && janas_llm_session_append(c->s, c->ids, (uint32_t)c->n_ids) == 0)
        for (int i = 0; i < 400; i++) { /* the recap, token by token */
            int32_t t;
            if (janas_llm_session_next(c->s, &t) != 0 ||
                janas_api_is_stop(c->llm, t))
                break;
        }
    else
        err = 1;
    c->closed = 0; /* the recap was cut off or ended: the next turn closes it */
    c->n_ids = 0;
    if (keep) {
        memcpy(c->ids, keep, n_keep * sizeof(int32_t));
        c->n_ids = n_keep;
        free(keep);
    }
    return err ? -1 : 0;
}

/*
 * Makes room for `need` more tokens by forgetting the oldest exchanges: the
 * system message stays, and whole turns go, so what is left reads as a
 * conversation. Returns 0 when there is room, -1 when even that is not
 * enough (a single message longer than the context).
 */
static int make_room(janas_llm_chat *c, uint32_t need)
{
    uint32_t cap = c->llm->n_ctx;
    uint32_t have = (uint32_t)janas_llm_session_length(c->s);
    if (have + need <= cap)
        return 0;
    /* the first tokens stay where they are: attention leans on them, and
       without that anchor the text after a slide drifts much further from
       what the same text would have given (measured with llm_slide: mean
       |logit difference| 2.7 with nothing kept, 0.5 with eight tokens) */
    uint32_t must = have + need - cap;
    uint32_t keep = c->sys_tokens ? c->sys_tokens : (have > 16 ? 8u : 0u);
    size_t k = 0;
    uint32_t drop = 0;
    while (k < c->n_turn) { /* oldest turns first, always whole */
        uint32_t end = c->turn[k] > keep ? c->turn[k] - keep : 0;
        k++;
        if (end > drop) {
            drop = end;
            if (drop >= must)
                break;
        }
    }
    if (drop < must) { /* nothing left to forget but the current turn */
        drop = have > keep ? have - keep : 0;
        k = c->n_turn;
    }
    if (!drop || janas_llm_session_slide(c->s, keep, drop) != 0)
        return -1;
    memmove(c->turn, c->turn + k, (c->n_turn - k) * sizeof(*c->turn));
    c->n_turn -= k;
    for (size_t i = 0; i < c->n_turn; i++)
        c->turn[i] -= drop;
    c->dropped += drop;
    return have + need - drop <= cap ? 0 : -1;
}

/* Records where this turn begins, for a later slide. */
int janas_api_note_turn(janas_llm_chat *c, uint32_t at)
{
    if (c->n_turn == c->cap_turn) {
        size_t cap = c->cap_turn ? 2 * c->cap_turn : 32;
        uint32_t *t = realloc(c->turn, cap * sizeof(*t));
        if (!t)
            return -1;
        c->turn = t;
        c->cap_turn = cap;
    }
    c->turn[c->n_turn++] = at;
    return 0;
}

void janas_api_unprepare(janas_llm_chat *c)
{
    if (!c->prepared)
        return;
    janas_llm_session_reset(c->s);
    c->prepared = 0;
    c->sys_tokens = 0;
}

int32_t janas_llm_chat_prepare(janas_llm_chat *c, int32_t *done, int32_t *total)
{
    if (done)
        *done = 0;
    if (total)
        *total = 0;
    if (!c)
        return janas_api_fail(JANAS_LLM_EINVAL, "no chat");
    if (!c->prepared) {
        if (janas_llm_session_length(c->s) > 0 || c->replying)
            return 0; /* the conversation has started */
        c->n_ids = c->n_text = 0;
        c->n_idpics = 0;
        int err = janas_api_begin_sequence(c);
        if (c->system || c->tools_sys || c->llm->format == FORMAT_GEMMA ||
            janas_api_think_note(c))
            err |= janas_api_add_system(c, c->system,
                                        c->system ? strlen(c->system) : 0);
        err |= janas_api_flush_text(c);
        if (err)
            return janas_api_fail(JANAS_LLM_ENOMEM, "out of memory");
        if (c->n_ids < 2)
            return 0; /* nothing worth reading ahead */
        c->sys_tokens = (uint32_t)c->n_ids;
        /* read before, in memory or on the disk: taken as it was */
        janas_api_keep_pick(c);
        const int32_t *h;
        uint32_t hn = janas_llm_session_tokens(c->s, &h), same = 0;
        while (same < hn && same < c->n_ids && h[same] == c->ids[same])
            same++;
        if (janas_llm_session_truncate(c->s, same) != 0) {
            janas_llm_session_reset(c->s); /* a recurrent state */
            same = 0;
        }
        if (janas_llm_session_append(c->s, c->ids + same,
                                     (uint32_t)c->n_ids - same) != 0) {
            janas_llm_session_reset(c->s);
            return janas_api_fail(JANAS_LLM_EFULL,
                                  "the system message does not fit in the "
                                  "context (%u tokens)",
                                  c->llm->n_ctx);
        }
        c->prepared = 1;
    }
    int r = 0;
    if (c->prepared == 1) {
        r = janas_llm_session_prefill_step(c->s);
        if (r < 0) {
            janas_api_unprepare(c);
            return janas_api_fail(JANAS_LLM_EFAIL,
                                  "the system message could not be read");
        }
        if (r == 0) {
            c->prepared = 2;
            janas_api_keep_prefix(c);
        }
    }
    /* all but the last token: the first message's pass reads it */
    uint32_t len = (uint32_t)janas_llm_session_length(c->s);
    uint32_t got = janas_llm_session_computed(c->s);
    if (done)
        *done = (int32_t)(got + 1 >= len ? len : got);
    if (total)
        *total = (int32_t)len;
    return r > 0 ? 1 : 0;
}

void janas_api_prep_start(janas_llm_chat *c)
{
    c->t_prep0 = janas_api_now();
    c->last_at = 0;
    c->stats.stage = JANAS_LLM_STAGE_IDLE;
}

void janas_api_parts(janas_llm_chat *c, uint32_t total)
{
    struct janas_llm_chat_stats *s = &c->stats;
    uint32_t sys = c->sys_tokens < total ? c->sys_tokens : total;
    uint32_t last = c->last_at < sys ? sys : c->last_at;
    if (last > total)
        last = total;
    s->prompt_system = sys;
    s->prompt_tools = c->tools_tokens < sys ? c->tools_tokens : sys;
    s->prompt_history = last - sys;
    s->prompt_last = total - last;
    s->prompt_tokens = total;
    s->context_size = c->llm->n_ctx;
}

int32_t janas_api_check_input(janas_llm_chat *c, uint32_t total)
{
    uint32_t max = c->p.max_input;
    if (!max || total <= max)
        return JANAS_LLM_OK;
    memset(&c->stats, 0, sizeof(c->stats));
    janas_api_parts(c, total);
    c->stats.input_tokens = total;
    c->stats.stage = JANAS_LLM_STAGE_IDLE;
    janas_metrics_refused(c, max);
    return janas_api_fail(JANAS_LLM_ELIMIT,
                          "the prompt is %u tokens, above the limit of %u "
                          "(max_input): %u too many",
                          total, max, total - max);
}

/* The reply starts: reply_at is where its turn begins in the sequence
   (UINT32_MAX for raw text), total the length of the prompt, cached how
   much of it was already computed. */
int janas_api_begin_reply(janas_llm_chat *c, uint32_t reply_at, uint32_t total,
                          uint32_t cached)
{
    c->replying = 1;
    c->closed = c->ended = 0;
    c->reply_tokens = 0;
    c->answer_tokens = 0;
    c->n_forced = c->i_forced = 0;
    c->npend = 0;
    c->n_rtext = 0;
    c->reply_at = reply_at;
    memset(&c->stats, 0, sizeof(c->stats));
    c->stats.input_tokens = total - cached;
    c->stats.cached_tokens = cached;
    janas_api_parts(c, total);
    /* the prompt is read in janas_llm_chat_next, a block at a time */
    c->stats.stage = JANAS_LLM_STAGE_INPUT;
    c->kv_from = janas_llm_session_computed(c->s);
    c->input_cpu = c->t_first = 0;
    c->eta_steps = 0;
    c->eta_w = c->eta_sx = c->eta_sc = c->eta_sxx = c->eta_sxc = 0;
    c->eta_last = -1;
    c->stats.prepare_seconds =
        c->t_send > c->t_prep0 ? c->t_send - c->t_prep0 : 0;
    janas_llm_session_mark(c->s); /* the penalties count from here */
    int rc = janas_api_reply_setup(c);
    janas_api_update_stats(c);
    janas_metrics_prompt(c);
    return rc;
}

/* The reply that just ended, remembered with its reasoning (see memo). */
void janas_api_memo_key(struct janas_buf *b, const char *text, size_t n,
                        const struct janas_tool_call *calls, size_t n_calls)
{
    while (n && (unsigned char)text[0] <= ' ') {
        text++;
        n--;
    }
    while (n && (unsigned char)text[n - 1] <= ' ')
        n--;
    janas_buf_put(b, text, n);
    for (size_t i = 0; i < n_calls; i++) {
        janas_buf_put(b, "\x1e", 1);
        janas_buf_put(b, calls[i].name, calls[i].name_n);
        janas_buf_put(b, "\x1f", 1);
        janas_buf_put(b, calls[i].args, calls[i].args_n);
    }
}

void janas_api_memo_store(janas_llm_chat *c)
{
    if (c->reply_at == UINT32_MAX)
        return;
    const int32_t *h;
    uint32_t hn = janas_llm_session_tokens(c->s, &h);
    if (c->reply_at >= hn)
        return;
    struct memo *m = &c->memo[c->memo_next];
    uint32_t n = hn - c->reply_at;
    int32_t *ids = malloc(n * sizeof(int32_t));
    struct janas_tool_call *calls =
        malloc((c->n_calls + 1) * sizeof(struct janas_tool_call));
    struct janas_buf key = {0};
    if (calls) {
        for (size_t i = 0; i < c->n_calls; i++)
            calls[i] =
                (struct janas_tool_call){.name = c->calls[i].name,
                                         .name_n = strlen(c->calls[i].name),
                                         .args = c->calls[i].args,
                                         .args_n = strlen(c->calls[i].args)};
        janas_api_memo_key(&key, c->rtext ? c->rtext : "", c->n_rtext, calls,
                           c->n_calls);
        janas_buf_put(&key, "", 0); /* a key of nothing is still a string */
    }
    free(calls);
    if (!ids || !calls || key.oom) {
        free(ids);
        janas_buf_free(&key);
        return;
    }
    memcpy(ids, h + c->reply_at, n * sizeof(int32_t));
    free(m->ids);
    free(m->text);
    *m = (struct memo){.text = key.p,
                       .n_text = key.n,
                       .ids = ids,
                       .n = n,
                       .closed = c->closed};
    c->memo_next =
        (c->memo_next + 1) % (int)(sizeof(c->memo) / sizeof(c->memo[0]));
}

int janas_api_rtext_add(janas_llm_chat *c, const char *s, size_t n)
{
    if (c->n_rtext + n > c->cap_rtext) {
        size_t cap = 2 * (c->n_rtext + n) + 256;
        char *t = realloc(c->rtext, cap);
        if (!t)
            return -1;
        c->rtext = t;
        c->cap_rtext = cap;
    }
    memcpy(c->rtext + c->n_rtext, s, n);
    c->n_rtext += n;
    return 0;
}

/*
 * One more turn of the conversation computed so far: the user's message
 * (texts NULL), or what the tools answered to the calls of the last reply,
 * n answers as one user turn, the way the chat templates write them.
 */
static int32_t send_turn(janas_llm_chat *c, const char *text, size_t n,
                         int32_t n_res, const char *const *texts,
                         const int32_t *lens)
{
    janas_api_prep_start(c);
    /* Gemma: the tools answer inside the model's turn, each answer named
       after its call, which the reply's calls say (cleared next) */
    int gemma_answers = texts && c->llm->format == FORMAT_GEMMA &&
                        c->llm->tools == JANAS_TOOLS_GEMMA &&
                        janas_llm_session_length(c->s) > 0;
    struct janas_buf ans = {0};
    for (int32_t i = 0; gemma_answers && i < n_res; i++) {
        const char *nm = i < (int32_t)c->n_calls ? c->calls[i].name : "tool";
        if (i > 0 || !c->closed)
            janas_buf_puts(&ans, "<|tool_response>");
        janas_tools_gemma_response(
            &ans, nm, strlen(nm), texts[i],
            janas_api_text_len(texts[i], lens ? lens[i] : -1));
    }
    janas_api_reply_clear(c);
    c->n_ids = c->n_text = 0;
    c->n_idpics = 0;
    int err = ans.oom;
    /* (ans is freed once written, below) */
    if (gemma_answers) {
        err |= janas_api_add_markup(c, ans.p, ans.n);
        c->in_turn = 1;
    } else if (c->prepared) {
        /* the system message, read ahead: the session holds it (the rest
           of it, when it was not read to the end, the reply reads) */
        c->prepared = 0;
    } else if (janas_llm_session_length(c->s) > 0) {
        /* close the previous reply: its stop token is in the sequence, or
           it was cut and gets one now; then the newline after it */
        if (!c->closed)
            err |= janas_api_add_special(c, c->llm->im_end);
        err |= janas_api_end_turn_nl(c);
    } else {
        /* the start: <bos> where the format has one, the system message */
        err |= janas_api_begin_sequence(c);
        if (c->system || c->tools_sys || c->llm->format == FORMAT_GEMMA ||
            janas_api_think_note(c))
            err |= janas_api_add_system(c, c->system,
                                        c->system ? strlen(c->system) : 0);
        c->sys_tokens = (uint32_t)c->n_ids;
    }
    err |= janas_api_note_turn(c, (uint32_t)janas_llm_session_length(c->s));
    janas_buf_free(&ans);
    if (gemma_answers) {
        /* written above */
    } else if (texts) {
        err |= janas_api_add_special(c, c->llm->im_start) ||
               janas_api_add_text(c, "user", 4);
        for (int32_t i = 0; i < n_res; i++) {
            err |= janas_api_add_markup(c, "\n<tool_response>\n", 17);
            err |= janas_api_add_text(
                c, texts[i], janas_api_text_len(texts[i], lens ? lens[i] : -1));
            err |= janas_api_add_markup(c, "\n</tool_response>", 17);
        }
        err |= janas_api_add_special(c, c->llm->im_end) ||
               janas_api_end_turn_nl(c);
    } else {
        /* the pictures queued: before the text, as the template has it */
        err |= janas_api_add_user(c, text, n, c->pics, c->n_pics);
    }
    err |= janas_api_flush_text(c);
    size_t reply_off = c->n_ids;
    err |= janas_api_start_reply(c);
    if (err)
        return janas_api_fail(JANAS_LLM_ENOMEM, "out of memory");
    /* the whole prompt the reply would read, against max_input: the
       conversation so far and this turn */
    c->last_at = (uint32_t)janas_llm_session_length(c->s);
    int32_t lim = janas_api_check_input(
        c, (uint32_t)janas_llm_session_length(c->s) + (uint32_t)c->n_ids);
    if (lim != JANAS_LLM_OK) {
        c->n_ids = 0;
        return lim;
    }
    if (c->llm->mode == JANAS_LLM_MODE_AUTO)
        janas_api_apply_mode(c->llm); /* the power source may have changed */
    /* the recap, if the context has to make room, runs before the clocks
       start: it is not part of this turn's speed */
    uint32_t want0 =
        (uint32_t)c->n_ids + (c->p.max_reply > 0 && c->p.max_reply < 1024
                                  ? (uint32_t)c->p.max_reply
                                  : 1024);
    if (c->p.recap && janas_llm_session_length(c->s) + want0 > c->llm->n_ctx)
        recap(c);
    janas_llm_session_stats(c->s, &c->st0);
    janas_expert_cache_stats(janas_llm_model_cache(c->llm->m), &c->cs0);
    c->t_send = janas_api_now();
    /* room for this message and for a reply of its own: rather than end the
       conversation when the context fills, the oldest exchanges are
       forgotten and the rest slides down */
    uint32_t reserve = c->p.max_reply > 0 && c->p.max_reply < 1024
                           ? (uint32_t)c->p.max_reply
                           : 1024;
    if (make_room(c, (uint32_t)c->n_ids + reserve) != 0) {
        c->replying = 0;
        return janas_api_fail(JANAS_LLM_EFULL,
                              "the context is full (%u tokens)", c->llm->n_ctx);
    }
    uint32_t at = (uint32_t)janas_llm_session_length(c->s);
    c->last_at = at; /* after the slide, if there was one */
    if (janas_llm_session_append(c->s, c->ids, (uint32_t)c->n_ids) != 0) {
        c->replying = 0;
        return janas_api_fail(JANAS_LLM_EFULL,
                              "the context is full (%u tokens)", c->llm->n_ctx);
    }
    /* the pictures' embeddings where their tokens are */
    if (c->n_idpics) {
        int bad = janas_api_media_register(c, at, 0);
        c->n_idpics = 0;
        janas_api_pics_free(c->pics, c->n_pics);
        c->pics = NULL;
        c->n_pics = 0;
        if (bad)
            return janas_api_fail(JANAS_LLM_ENOMEM, "out of memory");
    }
    if (janas_api_begin_reply(c, at + (uint32_t)reply_off,
                              at + (uint32_t)c->n_ids, at) != 0)
        return janas_api_fail(JANAS_LLM_ENOMEM, "out of memory");
    return JANAS_LLM_OK;
}

int32_t janas_llm_chat_send(janas_llm_chat *c, const char *text, int32_t len)
{
    if (!c || !text)
        return janas_api_fail(JANAS_LLM_EINVAL, "invalid argument");
    return send_turn(c, text, janas_api_text_len(text, len), 0, NULL, NULL);
}

int32_t janas_llm_chat_send_results(janas_llm_chat *c, int32_t n,
                                    const char *const *texts,
                                    const int32_t *lens)
{
    if (!c || n <= 0 || !texts)
        return janas_api_fail(JANAS_LLM_EINVAL, "invalid argument");
    for (int32_t i = 0; i < n; i++)
        if (!texts[i])
            return janas_api_fail(JANAS_LLM_EINVAL, "invalid argument");
    if (janas_llm_session_length(c->s) == 0)
        return janas_api_fail(JANAS_LLM_EINVAL,
                              "no reply called a tool: nothing to answer");
    if (c->llm->tools == JANAS_TOOLS_NONE)
        return janas_api_fail(JANAS_LLM_EMODEL,
                              "the model's chat template has no tools");
    return send_turn(c, NULL, 0, n, texts, lens);
}