37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
/*
* tools.c - the tools janas-mcp offers: generate, a question to the model
* running here, and embed, the vector of a text, each where a model for it
* was given.
*/
#include <math.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include "serve.h"
static void schema_str(struct janas_buf *b, const char *name, const char *what)
{
janas_buf_printf(b,
"\"%s\": {\"type\": \"string\", \"description\": ", name);
janas_json_write_str(b, what, strlen(what));
janas_buf_puts(b, "}");
}
void mcpd_tools_list(const struct mcpd *d, struct janas_buf *b)
{
janas_buf_puts(b, "\"tools\": [");
if (d->llm) {
char what[512];
snprintf(what, sizeof(what),
"Ask %s, a language model running on this machine: the "
"prompt and whatever it contains stay here. Returns its "
"answer. It knows only what the prompt tells it and what it "
"learnt in training; it has no tools and remembers nothing "
"between calls.",
d->name[0] ? d->name : "the local model");
janas_buf_puts(b, "{\"name\": \"generate\", \"title\": \"Ask the "
"local model\", \"description\": ");
janas_json_write_str(b, what, strlen(what));
janas_buf_puts(b, ", \"inputSchema\": {\"type\": \"object\", "
"\"properties\": {");
schema_str(b, "prompt", "What to ask, with everything it needs.");
janas_buf_puts(b, ", ");
schema_str(b, "system",
"Instructions for the model: its role, the "
"form of the answer (optional).");
janas_buf_puts(b, ", \"max_tokens\": {\"type\": \"integer\", "
"\"minimum\": 1, \"description\": \"The longest "
"answer, in tokens (a token is about three "
"quarters of a word).\"}, \"temperature\": "
"{\"type\": \"number\", \"minimum\": 0, "
"\"maximum\": 2, \"description\": \"0: always the "
"most likely word; higher: more varied.\"}}, "
"\"required\": [\"prompt\"], "
"\"additionalProperties\": false}}");
}
if (d->emb) {
char what[512];
snprintf(what, sizeof(what),
"The embedding of a text from %s, running on this machine: "
"a vector of %d numbers, of unit length, as a JSON array. "
"Texts of like meaning have vectors whose dot product is "
"near 1.",
d->emb_name[0] ? d->emb_name : "the local model",
janas_llm_embed_dim(d->emb));
janas_buf_puts(b, d->llm ? ", " : "");
janas_buf_puts(b, "{\"name\": \"embed\", \"title\": \"Embedding of "
"a text\", \"description\": ");
janas_json_write_str(b, what, strlen(what));
janas_buf_puts(b, ", \"inputSchema\": {\"type\": \"object\", "
"\"properties\": {");
schema_str(b, "text", "The text.");
janas_buf_puts(b, ", \"dimensions\": {\"type\": \"integer\", "
"\"minimum\": 1, \"description\": \"Fewer numbers: "
"the first ones, made unit length again "
"(optional).\"}}, \"required\": [\"text\"], "
"\"additionalProperties\": false}}");
}
janas_buf_puts(b, "]");
}
static double now(void)
{
struct timespec t;
clock_gettime(CLOCK_MONOTONIC, &t);
return (double)t.tv_sec + 1e-9 * (double)t.tv_nsec;
}
/* 1 when cancelled: nothing is answered then. */
static int generate(struct mcpd *d, const struct janas_json *args,
struct janas_buf *b)
{
const struct janas_json *prompt = janas_json_get(args, "prompt");
const struct janas_json *system = janas_json_get(args, "system");
const struct janas_json *max = janas_json_get(args, "max_tokens");
const struct janas_json *temp = janas_json_get(args, "temperature");
if (!janas_json_str(prompt)) {
const char *m = "prompt is required, as a string";
janas_mcps_text_result(b, m, strlen(m), 1);
return 0;
}
struct janas_llm_chat_params p = d->cp;
if (max && max->type == JANAS_JSON_NUMBER) {
double v = janas_json_num(max, 0);
p.max_reply = v >= 1 && v < 1e6 ? (int32_t)v : p.max_reply;
}
if (temp && temp->type == JANAS_JSON_NUMBER) {
double v = janas_json_num(temp, -1);
p.temperature = v >= 0 && v <= 2 ? (float)v : p.temperature;
}
int32_t rc = janas_llm_chat_set_params(d->chat, &p);
if (rc == JANAS_LLM_OK)
rc = janas_llm_chat_begin(d->chat);
if (rc == JANAS_LLM_OK && janas_json_str(system))
rc = janas_llm_chat_add(d->chat, JANAS_LLM_ROLE_SYSTEM, system->s,
(int32_t)system->n);
if (rc == JANAS_LLM_OK)
rc = janas_llm_chat_add(d->chat, JANAS_LLM_ROLE_USER, prompt->s,
(int32_t)prompt->n);
if (rc == JANAS_LLM_OK)
rc = janas_llm_chat_run(d->chat);
if (rc != JANAS_LLM_OK) {
const char *m = janas_llm_last_error();
janas_mcps_text_result(b, m, strlen(m), 1);
return 0;
}
struct janas_buf answer = {0};
char piece[256];
int32_t len;
double end = d->call_seconds > 0 ? now() + d->call_seconds : 0;
unsigned pieces = 0;
int late = 0;
while ((rc = janas_llm_chat_next(d->chat, piece, sizeof(piece), &len)) ==
JANAS_LLM_OK) {
if (!janas_llm_chat_thinking(d->chat)) /* the answer alone */
janas_buf_put(&answer, piece, (size_t)len);
if (++pieces % 8 == 0 && mcpd_poll(d)) {
janas_buf_free(&answer);
return 1;
}
if (end && now() > end) {
late = 1;
break;
}
}
/* the reasoning's markers stay out; the answer as the model ended it */
size_t from = 0;
while (from < answer.n && (answer.p[from] == '\n' || answer.p[from] == ' '))
from++;
if (answer.oom) {
const char *m = "out of memory";
janas_mcps_text_result(b, m, strlen(m), 1);
} else if (rc < 0 && answer.n == from) {
const char *m = janas_llm_last_error();
janas_mcps_text_result(b, m, strlen(m), 1);
} else {
if (late)
janas_buf_puts(&answer, "\n[cut: the time given to a call ran "
"out]");
else if (rc < 0)
janas_buf_printf(&answer, "\n[cut: %s]", janas_llm_last_error());
janas_mcps_text_result(b, answer.p ? answer.p + from : "",
answer.n - from, 0);
}
janas_buf_free(&answer);
return 0;
}
static void embed(struct mcpd *d, const struct janas_json *args,
struct janas_buf *b)
{
const struct janas_json *text = janas_json_get(args, "text");
const struct janas_json *dims = janas_json_get(args, "dimensions");
if (!janas_json_str(text)) {
const char *m = "text is required, as a string";
janas_mcps_text_result(b, m, strlen(m), 1);
return;
}
int32_t dim = janas_llm_embed_dim(d->emb), want = 0;
if (dims && dims->type == JANAS_JSON_NUMBER) {
double v = janas_json_num(dims, 0);
want = v >= 1 && v <= dim ? (int32_t)v : 0;
}
float *v = malloc((size_t)dim * sizeof(float));
int32_t n = 0, tokens = 0;
if (!v || janas_llm_embed(d->emb, text->s, (int32_t)text->n, want, v, &n,
&tokens) != JANAS_LLM_OK) {
const char *m = v ? janas_llm_last_error() : "out of memory";
janas_mcps_text_result(b, m, strlen(m), 1);
free(v);
return;
}
struct janas_buf a = {0};
janas_buf_puts(&a, "[");
for (int32_t i = 0; i < n; i++)
janas_buf_printf(&a, "%s%.7g", i ? ", " : "",
isfinite(v[i]) ? (double)v[i] : 0.0);
janas_buf_puts(&a, "]");
free(v);
if (a.oom) {
const char *m = "out of memory";
janas_mcps_text_result(b, m, strlen(m), 1);
} else {
janas_mcps_text_result(b, a.p, a.n, 0);
janas_buf_puts(b, ", \"structuredContent\": {\"embedding\": ");
janas_buf_put(b, a.p, a.n);
janas_buf_printf(b, ", \"tokens\": %d}", tokens);
}
janas_buf_free(&a);
}
int mcpd_tools_call(struct mcpd *d, const struct janas_json *params,
struct janas_buf *b)
{
const struct janas_json *name = janas_json_get(params, "name");
const struct janas_json *args = janas_json_get(params, "arguments");
if (args && args->type != JANAS_JSON_OBJECT)
args = NULL;
if (d->llm && janas_json_is(name, "generate"))
return generate(d, args, b);
if (d->emb && janas_json_is(name, "embed")) {
embed(d, args, b);
return 0;
}
return -1;
}