37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
/*
* request_opts.c - the parameters of a request to /chat/completions or
* /completions: sampling, stop strings, the number of replies, and what
* only chat has - tools, the form of the answer, log-probabilities.
*/
#include "request.h"
#include <stdlib.h>
#include <string.h>
/* logit_bias: token ids (as the object's keys) to a bias of -100..100 */
static int read_bias(struct srv_perr *e, struct srv_parsed *pr, yyjson_val *lb,
struct srv_job *j)
{
if (!lb || yyjson_is_null(lb))
return 0;
if (!yyjson_is_obj(lb))
return srv_pfail(
pr, BAD("invalid_value", "'logit_bias' must be an object."));
size_t n = yyjson_obj_size(lb);
if (n > 1024)
return srv_pfail(pr, BAD("invalid_value",
"'logit_bias' holds at most 1024 tokens."));
j->bias_id = malloc((n + 1) * sizeof(int32_t));
j->bias = malloc((n + 1) * sizeof(float));
if (!j->bias_id || !j->bias)
return -1;
size_t idx, max;
yyjson_val *k, *v;
yyjson_obj_foreach(lb, idx, max, k, v)
{
char *end;
long id = strtol(yyjson_get_str(k), &end, 10);
double x;
if (*end || id < 0 || id > INT32_MAX || srv_num(v, -100, 100, &x) != 1)
return srv_pfail(pr, BAD("invalid_value",
"'logit_bias' maps token ids to numbers "
"from -100 to 100."));
j->bias_id[j->n_bias] = (int32_t)id;
j->bias[j->n_bias++] = (float)x;
}
return 0;
}
static int read_stop(struct srv_perr *e, struct srv_parsed *pr, yyjson_val *st,
struct srv_job *j)
{
if (st && yyjson_is_str(st) && yyjson_get_len(st) > 0) {
j->stop[j->n_stop++] =
srv_dup_n(yyjson_get_str(st), yyjson_get_len(st));
} else if (st && yyjson_is_arr(st)) {
size_t idx, max;
yyjson_val *s;
if (yyjson_arr_size(st) > SRV_MAX_STOP)
return srv_pfail(pr, BAD("invalid_value",
"'stop' holds at most sixteen strings."));
yyjson_arr_foreach(st, idx, max, s)
{
if (!yyjson_is_str(s))
return srv_pfail(
pr, BAD("invalid_value", "'stop' must hold strings."));
if (yyjson_get_len(s) > 0)
j->stop[j->n_stop++] =
srv_dup_n(yyjson_get_str(s), yyjson_get_len(s));
}
} else if (st && !yyjson_is_null(st) && !yyjson_is_str(st)) {
return srv_pfail(
pr, BAD("invalid_value", "'stop' must be a string or an array."));
}
for (int i = 0; i < j->n_stop; i++)
if (!j->stop[i])
return -1;
return 0;
}
/* store and metadata: the completion kept for GET /chat/completions */
static int read_store(struct srv_perr *e, struct srv_parsed *pr,
yyjson_val *root)
{
yyjson_val *st = yyjson_obj_get(root, "store");
pr->store = st && yyjson_is_bool(st) && yyjson_get_bool(st);
yyjson_val *md = yyjson_obj_get(root, "metadata");
if (!md || yyjson_is_null(md))
return 0;
if (!yyjson_is_obj(md) || yyjson_obj_size(md) > 16)
return srv_pfail(pr, BAD("invalid_value",
"'metadata' is an object of at most 16 "
"strings."));
size_t idx, max;
yyjson_val *k, *v;
yyjson_obj_foreach(md, idx, max, k, v)
{
if (!yyjson_is_str(v) || yyjson_get_len(k) > 64 ||
yyjson_get_len(v) > 512)
return srv_pfail(pr, BAD("invalid_value",
"'metadata': keys of at most 64 "
"characters, string values of at most "
"512."));
}
return (pr->metadata = srv_json_text(md, NULL)) ? 0 : -1;
}
int srv_read_params(struct srv_perr *e, struct srv_parsed *pr, yyjson_val *root,
struct srv_job *j, int chat)
{
double x;
int got;
if ((got = srv_num(yyjson_obj_get(root, "temperature"), 0, 2, &x)) < 0)
return srv_pfail(
pr, BAD("invalid_value", "'temperature' must be between 0 and 2."));
if (got)
j->p.temperature = (float)x;
if ((got = srv_num(yyjson_obj_get(root, "top_p"), 0, 1, &x)) < 0)
return srv_pfail(
pr, BAD("invalid_value", "'top_p' must be between 0 and 1."));
if (got)
j->p.top_p = (float)x;
if ((got = srv_num(yyjson_obj_get(root, "top_k"), 0, 1e6, &x)) < 0)
return srv_pfail(pr,
BAD("invalid_value", "'top_k' must be 0 or more."));
if (got)
j->p.top_k = (int32_t)x;
if ((got = srv_num(yyjson_obj_get(root, "min_p"), 0, 1, &x)) < 0)
return srv_pfail(
pr, BAD("invalid_value", "'min_p' must be between 0 and 1."));
if (got)
j->p.min_p = (float)x;
static const char *const pen[] = {"presence_penalty", "frequency_penalty"};
for (int i = 0; i < 2; i++) {
if ((got = srv_num(yyjson_obj_get(root, pen[i]), -2, 2, &x)) < 0)
return srv_pfail(pr, BAD("invalid_value",
"'%s' must be between -2 and 2.", pen[i]));
if (got && i == 0)
j->p.presence_penalty = (float)x;
if (got && i == 1)
j->p.frequency_penalty = (float)x;
}
/*
* max_completion_tokens counts the reasoning, as OpenAI defines it. The
* older max_tokens of a chat is taken for the answer alone: OpenAI does
* not take it for the models that reason, and a client that sends it
* (Open WebUI asks for a title with 1000) means the answer - counted
* with the reasoning, a model that reasons at length spent it all
* before writing a word. In completions there is no reasoning.
*/
int total = !chat || yyjson_obj_get(root, "max_completion_tokens");
const char *mt = yyjson_obj_get(root, "max_completion_tokens")
? "max_completion_tokens"
: "max_tokens";
if ((got = srv_num(yyjson_obj_get(root, mt), 1, 1e9, &x)) < 0)
return srv_pfail(pr,
BAD("invalid_value", "'%s' must be 1 or more.", mt));
if (got && !total)
j->p.max_answer = (int32_t)x;
else
j->p.max_reply = got ? (int32_t)x : chat ? 0 : 16; /* OpenAI's */
yyjson_val *seed = yyjson_obj_get(root, "seed");
if (seed && yyjson_is_int(seed)) {
j->p.seed = (uint64_t)yyjson_get_sint(seed)
? (uint64_t)yyjson_get_sint(seed)
: 1;
j->seeded = 1;
}
/* n replies; completions: best_of candidates, the best n returned */
if ((got = srv_num(yyjson_obj_get(root, "n"), 1, 128, &x)) < 0 ||
(got && x != (int)x))
return srv_pfail(pr, BAD("invalid_value",
"'n' must be a whole number from 1 to 128."));
int n = got ? (int)x : 1, gen = n;
if (!chat) {
if ((got = srv_num(yyjson_obj_get(root, "best_of"), 1, 128, &x)) < 0 ||
(got && (x != (int)x || x < n)))
return srv_pfail(pr, BAD("invalid_value",
"'best_of' must be a whole number from "
"n to 128."));
if (got)
gen = (int)x;
}
if (read_bias(e, pr, yyjson_obj_get(root, "logit_bias"), j) != 0 ||
read_stop(e, pr, yyjson_obj_get(root, "stop"), j) != 0 ||
read_store(e, pr, root) != 0)
return -1;
yyjson_val *sv = yyjson_obj_get(root, "stream");
pr->stream = sv && yyjson_is_bool(sv) && yyjson_get_bool(sv);
yyjson_val *so = yyjson_obj_get(root, "stream_options");
yyjson_val *iu = so ? yyjson_obj_get(so, "include_usage") : NULL;
pr->include_usage = iu && yyjson_is_bool(iu) && yyjson_get_bool(iu);
if (!chat) {
yyjson_val *lp = yyjson_obj_get(root, "logprobs");
if ((got = srv_num(lp, 0, 20, &x)) < 0)
return srv_pfail(
pr, BAD("invalid_value", "'logprobs' must be from 0 to 20."));
if (got) { /* the token's own, and x alternatives */
pr->logprobs = (int)x + 1;
j->p.logprobs = 1;
j->p.top_logprobs = (int32_t)x;
}
yyjson_val *ec = yyjson_obj_get(root, "echo");
pr->echo = ec && yyjson_is_bool(ec) && yyjson_get_bool(ec);
yyjson_val *sx = yyjson_obj_get(root, "suffix");
if (sx && yyjson_is_str(sx)) {
j->n_suffix = yyjson_get_len(sx);
if (!(j->suffix = srv_dup_n(yyjson_get_str(sx), j->n_suffix)))
return -1;
} else if (sx && !yyjson_is_null(sx)) {
return srv_pfail(
pr, BAD("invalid_value", "'suffix' must be a string."));
}
if (gen > n) /* the best are chosen by their log-probability */
j->p.logprobs = 1;
}
if (pr->stream && gen > n)
return srv_pfail(pr, BAD("invalid_value",
"'best_of' cannot be streamed: the best is "
"known only at the end."));
return srv_job_choices(j, gen, n);
}
/* tool_choice, or the older function_call */
static int read_choice(struct srv_perr *e, struct srv_parsed *pr,
yyjson_val *tc, struct srv_job *j)
{
if (!tc || yyjson_is_null(tc))
return 0;
const char *s = yyjson_get_str(tc);
if (s) {
j->tool_choice = strcmp(s, "none") == 0 ? JANAS_LLM_TOOLS_NONE
: strcmp(s, "required") == 0 ? JANAS_LLM_TOOLS_REQUIRED
: strcmp(s, "auto") == 0 ? JANAS_LLM_TOOLS_AUTO
: -1;
if (j->tool_choice < 0)
return srv_pfail(pr, BAD("invalid_value",
"'tool_choice' is none, auto, required "
"or a function."));
return 0;
}
yyjson_val *fn = yyjson_obj_get(tc, "function");
const char *name = yyjson_get_str(fn ? yyjson_obj_get(fn, "name")
: yyjson_obj_get(tc, "name"));
if (!name || strlen(name) >= sizeof(j->tool_name))
return srv_pfail(
pr, BAD("invalid_value", "'tool_choice' names no function."));
j->tool_choice = JANAS_LLM_TOOLS_FUNCTION;
memcpy(j->tool_name, name, strlen(name) + 1);
return 0;
}
/* tools, or the older functions (the functions alone) */
static int read_tools(struct srv_perr *e, struct srv_parsed *pr,
yyjson_val *root, struct srv_job *j)
{
yyjson_val *tools = yyjson_obj_get(root, "tools");
if (!tools || yyjson_is_null(tools))
tools = yyjson_obj_get(root, "functions");
if (!tools || yyjson_is_null(tools) ||
(yyjson_is_arr(tools) && yyjson_arr_size(tools) == 0))
return 0;
if (!yyjson_is_arr(tools))
return srv_pfail(pr, BAD("invalid_value", "'tools' must be an array."));
size_t idx, max;
yyjson_val *t;
yyjson_arr_foreach(tools, idx, max, t)
{
yyjson_val *fn = yyjson_obj_get(t, "function");
const char *type = yyjson_get_str(yyjson_obj_get(t, "type"));
if (type && strcmp(type, "function") != 0)
return srv_pfail(pr, BAD("unsupported_value",
"tools[%zu]: only functions can be "
"called here (got '%s').",
idx, type));
if (!yyjson_get_str(yyjson_obj_get(fn ? fn : t, "name")))
return srv_pfail(pr, BAD("invalid_value",
"tools[%zu] has no function name.", idx));
}
if (!(j->tools = srv_json_text(tools, &j->n_tools)))
return -1;
yyjson_val *pc = yyjson_obj_get(root, "parallel_tool_calls");
if (pc && yyjson_is_bool(pc))
j->parallel = yyjson_get_bool(pc);
yyjson_val *tc = yyjson_obj_get(root, "tool_choice");
if (!tc || yyjson_is_null(tc))
tc = yyjson_obj_get(root, "function_call");
return read_choice(e, pr, tc, j);
}
/* response_format: text, json_object, or json_schema */
static int read_format(struct srv_perr *e, struct srv_parsed *pr,
yyjson_val *rf, struct srv_job *j)
{
if (!rf || yyjson_is_null(rf))
return 0;
const char *type = yyjson_get_str(yyjson_obj_get(rf, "type"));
if (!type)
return srv_pfail(
pr, BAD("invalid_value", "'response_format' needs a type."));
if (strcmp(type, "text") == 0)
return 0;
if (strcmp(type, "json_object") == 0) {
j->format = JANAS_LLM_FORMAT_JSON;
return 0;
}
if (strcmp(type, "json_schema") != 0)
return srv_pfail(pr, BAD("invalid_value",
"'response_format' is text, json_object or "
"json_schema (got '%s').",
type));
yyjson_val *js = yyjson_obj_get(rf, "json_schema");
yyjson_val *schema = js ? yyjson_obj_get(js, "schema") : NULL;
if (!schema || !(yyjson_is_obj(schema) || yyjson_is_bool(schema)))
return srv_pfail(pr, BAD("invalid_value",
"'response_format.json_schema.schema' must "
"be a JSON Schema."));
j->format = JANAS_LLM_FORMAT_SCHEMA;
return (j->schema = srv_json_text(schema, &j->n_schema)) ? 0 : -1;
}
int srv_read_chat_opts(struct srv_perr *e, struct srv_parsed *pr,
yyjson_val *root, struct srv_job *j)
{
if (read_tools(e, pr, root, j) != 0 ||
read_format(e, pr, yyjson_obj_get(root, "response_format"), j) != 0)
return -1;
yyjson_val *lp = yyjson_obj_get(root, "logprobs");
if (lp && yyjson_is_bool(lp) && yyjson_get_bool(lp)) {
pr->logprobs = 1;
j->p.logprobs = 1;
}
double x;
int got = srv_num(yyjson_obj_get(root, "top_logprobs"), 0, 20, &x);
if (got < 0 || (got && !pr->logprobs && x > 0))
return srv_pfail(pr, BAD("invalid_value",
"'top_logprobs' is from 0 to 20, and asks "
"for 'logprobs': true."));
if (got)
j->p.top_logprobs = (int32_t)x;
yyjson_val *mod = yyjson_obj_get(root, "modalities");
if (mod && yyjson_is_arr(mod)) {
size_t idx, max;
yyjson_val *v;
yyjson_arr_foreach(mod, idx, max, v)
{
const char *s = yyjson_get_str(v);
if (!s || strcmp(s, "text") != 0)
return srv_pfail(pr,
BAD("unsupported_value",
"Only the text modality is supported."));
}
}
/* the reasoning: OpenAI's reasoning_effort, or the chat template's own
switch as other servers take it */
const char *eff = yyjson_get_str(yyjson_obj_get(root, "reasoning_effort"));
if (eff)
j->p.thinking =
strcmp(eff, "none") == 0 || strcmp(eff, "minimal") == 0 ? 0 : 1;
yyjson_val *kw = yyjson_obj_get(root, "chat_template_kwargs");
yyjson_val *et = kw ? yyjson_obj_get(kw, "enable_thinking") : NULL;
if (et && yyjson_is_bool(et))
j->p.thinking = yyjson_get_bool(et) ? 1 : 0;
if (srv_reasoning_budget(root, &j->p) != 0)
return srv_pfail(
pr, BAD("invalid_value",
"'reasoning_budget_tokens' is -1 (no limit), 0 (no "
"reasoning) or a number of tokens."));
return 0;
}
void srv_reasoning_default(const struct srv_config *cfg,
struct janas_llm_chat_params *p)
{
p->thinking = cfg->thinking;
if (cfg->reasoning_budget == 0)
p->thinking = 0;
else if (cfg->reasoning_budget > 0)
p->max_reasoning = cfg->reasoning_budget;
}
int srv_reasoning_budget(yyjson_val *root, struct janas_llm_chat_params *p)
{
yyjson_val *b = yyjson_obj_get(root, "reasoning_budget_tokens");
if (!b)
b = yyjson_obj_get(root, "thinking_budget_tokens");
double x;
int got = srv_num(b, -1, 2147483647.0, &x);
if (got < 0 || (got && x != (int32_t)x))
return -1;
if (got && x == 0) {
p->thinking = 0;
p->max_reasoning = 0;
} else if (got)
p->max_reasoning = x < 0 ? 0 : (int32_t)x;
return 0;
}