37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
/*
* server.h - janas-server: the model behind an HTTP API that follows
* OpenAI's (https://github.com/openai/openai-openapi).
*
* Three parts. http.c takes the requests (libmicrohttpd, a thread per
* connection), finds the route in the table generated from the
* specification (routes.inc) and answers. engine.c owns the model: one
* thread runs the requests one at a time, in the order they came, and hands
* the text over as it is written. The handlers (models.c, chat.c) read the
* JSON of a request into a job and write the JSON of the answer.
*/
#ifndef JANAS_SERVER_H
#define JANAS_SERVER_H
#include <pthread.h>
#include <stdatomic.h>
#include <stddef.h>
#include <stdint.h>
#include "common/mcp_policy.h"
#include "janas/llm.h"
#include "store.h"
#include <yyjson.h>
struct MHD_Connection;
struct srv_config {
const char *model_path;
struct janas_llm_params llm; /* how the model is opened */
int no_gpu;
int no_mtp; /* no MTP file, not even one found beside the model */
const char *model_id; /* the name clients use; default: the file name */
const char *host; /* address to listen on (default 127.0.0.1) */
uint16_t port; /* default 8080 */
const char *api_key; /* NULL: none asked */
uint32_t max_queue; /* requests waiting at most (default 16) */
size_t max_body; /* bytes of a request body at most */
int max_connections;
int verbose;
int test; /* serve the test chat page at /test */
int thinking; /* reasoning when a request does not say: 1, 0, or -1
for the model's own default */
int reasoning_budget; /* the longest reasoning when a request does not
say, in tokens: -1 no limit (default), 0 none */
const char *embed_path; /* the embedding model, or NULL */
const char *embed_id; /* its name for clients */
const char *speech_path; /* the speech-to-text model, or NULL */
const char *vad_path; /* its voice activity detector, or NULL */
const char *speaker_path; /* its speaker embeddings, or NULL */
const char *speech_id; /* its name for clients */
const char *store_dir; /* where stored documents outlive the server;
NULL: memory only (--no-store) */
int keep; /* conversations kept computed besides the
current one (default 8) */
uint64_t keep_bytes; /* memory for them; 0: automatic */
uint64_t keep_disk; /* disk for them, below the memory; 0: none
(default 8 GiB) */
int no_web; /* refuse the web_search tool (Responses) */
int code_interpreter; /* run code (code_interpreter, containers) */
const char *code_python; /* the python3 of the cage, NULL: the system's */
const char *tls_cert, *tls_key; /* HTTPS (tls.c); NULL: HTTP */
int no_mcp; /* refuse the tools of type mcp (Responses):
no connection goes out on a request's word */
struct janas_net_policy mcp_policy; /* where they may: public addresses,
and those --mcp-allow names */
const char **allow_origin; /* pages of other origins let in
(--allow-origin; "*": any) */
int n_allow_origin;
uint32_t max_input; /* longest prompt taken, tokens; 0: the context */
uint32_t warn_input; /* prompts longer are taken, and said on stderr */
uint32_t fim_prefix; /* fill in the middle: the tokens before the
cursor read at most (0: all) */
uint32_t fim_suffix; /* the lines after it (0: all) */
uint32_t fim_max; /* the tokens it writes at most (0: as asked) */
};
/* Stop strings a request may give: OpenAI takes four, editors' extensions
send more (Continue up to sixteen). */
#define SRV_MAX_STOP 16
/* The engine: the model, its conversations, and the queue. */
struct srv_engine;
/* JOB_COUNT: only how many tokens the messages are (j->prompt_tokens) */
enum job_kind { JOB_CHAT, JOB_TEXT, JOB_COUNT, JOB_EMBED };
/* The server's reasoning for a request that does not say (--think,
--reasoning-budget), into p. */
void srv_reasoning_default(const struct srv_config *cfg,
struct janas_llm_chat_params *p);
/* A message of a chat request, with the tool calls of an assistant's. */
/* A picture of a message (images.c): its reference as the request gave it
(a data: or http(s) URL, or a file id), and its bytes once fetched. */
/* a data: URL's bytes (base64) of at most max: 0, or -1 with the reason */
int srv_data_url(const char *url, size_t len, size_t max, unsigned char **data,
size_t *n_data, char *err, size_t el);
struct srv_img {
char *ref;
int is_file;
unsigned char *data;
size_t n;
};
struct srv_msg {
int32_t role; /* JANAS_LLM_ROLE_... */
char *text;
size_t n;
int n_calls;
char **call_name, **call_args;
struct srv_img *imgs; /* a user message's pictures, before its text */
int n_imgs;
};
/* images.c: a part that is a picture (chat's image_url, the Responses'
input_image) added to imgs: 1, 0 when the part is not a picture, -1 with
the reason; the pictures of a job's messages fetched (0, or -1 with the
reason); a list freed */
int srv_image_part(yyjson_val *part, const char *type, struct srv_img **imgs,
int *n_imgs, char *err, size_t el);
struct srv_job;
struct srv;
int srv_images_fetch(struct srv *s, struct srv_job *j, char *err, size_t el);
void srv_imgs_free(struct srv_img *imgs, int n);
/* A prompt of a completion: text, or token ids (the engine turns them
into text, which is where an echo finds it). */
struct srv_prompt {
char *text;
size_t n;
int32_t *ids;
size_t n_ids;
};
/* A token of the answer with its log-probability, and the alternatives. */
struct srv_alt {
char *tok; /* its bytes */
size_t n;
float lp;
};
struct srv_lp {
struct srv_alt t;
int n_top;
struct srv_alt *top;
};
/* A tool call of the reply. */
struct srv_call {
char id[40];
char *name, *args;
};
/* One of the replies a request asks for (n, best_of). */
struct srv_choice {
char *think, *text; /* the reasoning and the answer as they grow */
size_t n_think, cap_think, n_text, cap_text;
struct srv_call *calls;
int n_calls, cap_calls;
struct srv_lp *lp; /* the answer's tokens, when logprobs are asked */
size_t n_lp, cap_lp;
int by_stop; /* ended on one of the job's stop strings */
int32_t finish; /* JANAS_LLM_FINISH_... */
int done;
double score; /* the mean log-probability, for best_of */
uint32_t tokens, reasoning_tokens;
};
/* A request for the model, filled by a handler, run by the engine. */
struct srv_job {
enum job_kind kind;
/* JOB_CHAT: the messages; JOB_TEXT: the texts to continue (or to fill
in before suffix), n_choices / n_prompts replies each */
struct srv_msg *msg;
int32_t n_msg;
struct srv_prompt *prompts;
int n_prompts;
char *suffix;
size_t n_suffix;
/* /infill, as llama.cpp: a new line indented less than n_indent ends
the text; past t_max_predict_ms, so does a new line; prefill_only:
the prompt is read, nothing is answered */
int n_indent, prefill_only;
int32_t t_max_predict_ms;
struct janas_llm_chat_params p;
int seeded; /* the request gave a seed: choice k takes seed + k */
char *stop[SRV_MAX_STOP]; /* strings that end the reply where they
appear */
int n_stop;
/* tools (a JSON array, as the request has it), tool_choice, and the
form of the answer */
char *tools;
size_t n_tools;
int32_t tool_choice;
char tool_name[128];
int parallel;
int32_t format;
char *schema;
size_t n_schema;
int32_t *bias_id;
float *bias;
int32_t n_bias;
int32_t dims; /* JOB_EMBED: the vectors' length (0: the model's) */
float *emb; /* JOB_EMBED: n_prompts vectors of emb_dim, the engine's */
int32_t emb_dim;
/* Responses, tools of type mcp: the outputs of the calls approved in
this request, by the id of the approval request */
char **mcp_id, **mcp_out;
int n_mcp;
int n_choices; /* replies to generate */
int n_keep; /* of which to return (best_of: the best n_keep) */
/* what the engine writes, under mu */
pthread_mutex_t mu;
pthread_cond_t cv;
struct srv_choice *ch; /* n_choices of them */
int cur; /* the one being written */
int started; /* the engine took it */
int done; /* it is over */
int cancel; /* the client went away: stop */
int32_t err; /* JANAS_LLM_OK or the library's code */
char errmsg[256];
struct janas_llm_chat_stats st; /* of the first reply */
uint32_t prompt_tokens; /* of every prompt, once */
uint32_t completion_tokens, reasoning_tokens; /* of all of them */
uint32_t queue_pos; /* requests ahead of it, while it waits */
int low; /* a batch's: every other request goes first */
const _Atomic int *watch; /* its submitter's (srv_engine_watch): 1 stops
it as cancel does */
int refs; /* the handler and the engine: freed by the last one */
struct srv_job *next;
};
struct srv_engine *srv_engine_start(const struct srv_config *cfg, char *err,
size_t err_len);
void srv_engine_stop(struct srv_engine *e);
/* At stop, before the connections close: the clients' requests, running
and waiting, end at their next token, and no new one is taken, so no
connection waits for a whole reply that no one will read; the batches'
and the evals' go on (their workers stop them where they resume). */
void srv_engine_cancel_clients(struct srv_engine *e);
/* Whether an embedding model is open, and the length of its vectors. */
int32_t srv_engine_embed_dim(const struct srv_engine *e);
/* The model's description (janas_llm_describe), for /v1/models. */
const char *srv_engine_describe(const struct srv_engine *e);
int64_t srv_engine_created(const struct srv_engine *e);
/* The system message janas-chat gives the model (janas_llm_default_system),
offered to the test page. */
const char *srv_engine_system(const struct srv_engine *e);
struct srv_job *srv_job_new(void);
void srv_job_release(struct srv_job *j);
/* The choices the job's replies go into: 0, or -1 on memory. */
int srv_job_choices(struct srv_job *j, int n, int keep);
/* Queues the job; -1 when the queue is full. A job submitted on a thread
that set srv_engine_low (the batches') waits behind every other. */
int srv_engine_submit(struct srv_engine *e, struct srv_job *j);
extern _Thread_local int srv_engine_low;
/* A flag the jobs submitted on this thread watch: set to 1 elsewhere, they
end at their next token (a run of the Assistants API cancelled). */
extern _Thread_local const _Atomic int *srv_engine_watch;
/* How much of choice k's answer may be handed out now: while the reply
runs, the end that could still turn out to be the start of a stop string
is held back. Call with j->mu held. */
size_t srv_job_visible(const struct srv_job *j, int k);
/* The server: configuration, engine, and the HTTP side. */
struct srv {
struct srv_config cfg;
struct srv_engine *engine;
struct srv_store *store; /* completions, responses, conversations */
struct srv_files *files; /* uploaded files (files_store.h) */
struct MHD_Daemon *mhd;
janas_llm_speech *speech; /* --speech-model, or NULL */
pthread_mutex_t speech_mu; /* one transcription at a time */
};
struct sockaddr;
/* tls.c: the certificate and key read (0, or -1 with the reason), the
listener of HTTPS on addr, and its end */
int srv_tls_setup(const char *cert, const char *key, char *err, size_t err_len);
int srv_tls_start(struct srv *s, const struct sockaddr *addr, char *err,
size_t err_len);
void srv_tls_stop(void);
/* Listening on addr (the port inside it), and no longer. */
int srv_http_start(struct srv *s, const struct sockaddr *addr, char *err,
size_t err_len);
void srv_http_stop(struct srv *s);
/* One request, as the handlers see it. */
/* An answer kept instead of sent: a request the server makes of itself
(a batch's lines) */
struct srv_capture {
unsigned status;
char *body; /* malloc'd, NUL-terminated */
size_t n;
/* when set, a stream's bytes go to it as they come instead of into
body (a run of the Assistants API reading /responses streamed) */
void (*on_data)(void *arg, const char *p, size_t n);
void *arg;
};
struct srv_req {
struct srv *srv;
struct MHD_Connection *conn; /* NULL with cap */
struct srv_capture *cap; /* the answer goes here when set */
const char *method, *path; /* path without /v1 */
char param[3][256]; /* the {…} segments of the route, in order */
int n_param;
const char *body;
size_t n_body;
};
/* A handler answers through the functions below and returns their result
(an MHD_Result). */
typedef int (*srv_handler)(struct srv_req *r);
/* The handler of method and path (without /v1), or NULL. */
srv_handler srv_route_find(const char *method, const char *path);
/* Answers: JSON text (taken over, freed here), an OpenAI error, or a
stream of server-sent events. */
int srv_reply_json(struct srv_req *r, unsigned status, char *json, size_t n);
int srv_reply_error(struct srv_req *r, unsigned status, const char *type,
const char *code, const char *fmt, ...)
__attribute__((format(gnu_printf, 5, 6)));
/* A job refused for an input above --max-input: a 400 with the numbers
(input_tokens, max_input_tokens, excess_tokens and the parts) as fields
of the error, for a client to shorten the input by. */
struct srv_job;
int srv_reply_limit(struct srv_req *r, const struct srv_job *j);
/* The stream: next(ctx, buf, cap) writes up to cap bytes and returns how
many, 0 when it has nothing yet (it may wait a little first), -1 at the
end; done(ctx) is called once the connection is over. */
typedef long (*srv_stream_next)(void *ctx, char *buf, size_t cap);
typedef void (*srv_stream_done)(void *ctx);
/* n bytes (taken over, freed here) of that Content-Type as the answer. */
int srv_reply_bytes(struct srv_req *r, unsigned status, char *body, size_t n,
const char *type);
/* n bytes of an open file (taken over, closed here) as the answer. */
int srv_reply_fd(struct srv_req *r, int fd, uint64_t n, const char *type);
int srv_reply_stream(struct srv_req *r, srv_stream_next next,
srv_stream_done done, void *ctx);
/* The handlers */
int srv_models_list(struct srv_req *r);
int srv_models_get(struct srv_req *r);
int srv_models_delete(struct srv_req *r);
int srv_embeddings_create(struct srv_req *r);
/* audio.c: speech to text with the speech model (--speech-model) */
int srv_audio_transcriptions(struct srv_req *r);
int srv_audio_translations(struct srv_req *r);
int srv_speech_start(struct srv *s, char *err, size_t err_len);
void srv_speech_stop(struct srv *s);
int srv_moderations_create(struct srv_req *r);
int srv_chat_create(struct srv_req *r);
int srv_completion_create(struct srv_req *r);
int srv_infill(struct srv_req *r); /* POST /infill, llama.cpp's */
/* stored.c: the completions kept with store: true */
int srv_stored_list(struct srv_req *r);
int srv_stored_get(struct srv_req *r);
int srv_stored_update(struct srv_req *r);
int srv_stored_delete(struct srv_req *r);
int srv_stored_messages(struct srv_req *r);
/* files.c: the Files API */
int srv_files_create(struct srv_req *r);
int srv_files_list_req(struct srv_req *r);
int srv_files_retrieve(struct srv_req *r);
int srv_files_delete_req(struct srv_req *r);
int srv_files_content(struct srv_req *r);
/* uploads.c: the Uploads API; a part's body may pass --max-body up to
SRV_UPLOAD_PART_MAX and a little for its envelope */
#define SRV_UPLOAD_PART_MAX ((size_t)64 << 20)
int srv_uploads_create(struct srv_req *r);
int srv_uploads_part(struct srv_req *r);
int srv_uploads_complete(struct srv_req *r);
int srv_uploads_cancel(struct srv_req *r);
void srv_uploads_shutdown(void);
/* batches.c: the Batch API, and the thread that runs the batches (started
once the store, the files and the engine are there) */
int srv_batches_create(struct srv_req *r);
int srv_batches_get(struct srv_req *r);
int srv_batches_cancel(struct srv_req *r);
int srv_batches_list(struct srv_req *r);
int srv_batches_start(struct srv *s);
void srv_batches_stop(void);
/* evals.c: the Evals API, and the thread that runs the evals' runs */
int srv_evals_create(struct srv_req *r);
int srv_evals_list(struct srv_req *r);
int srv_evals_get(struct srv_req *r);
int srv_evals_update(struct srv_req *r);
int srv_evals_delete(struct srv_req *r);
int srv_eval_runs_list(struct srv_req *r);
int srv_eval_runs_create(struct srv_req *r);
int srv_eval_runs_get(struct srv_req *r);
int srv_eval_runs_cancel(struct srv_req *r);
int srv_eval_runs_delete(struct srv_req *r);
int srv_eval_items_list(struct srv_req *r);
int srv_eval_items_get(struct srv_req *r);
int srv_evals_start(struct srv *s);
void srv_evals_stop(void);
/* containers.c: the Containers API (containers.h), and the sessions of
code_interpreter, ended at stop */
int srv_containers_create(struct srv_req *r);
int srv_containers_list(struct srv_req *r);
int srv_containers_get(struct srv_req *r);
int srv_containers_delete(struct srv_req *r);
int srv_container_files_create(struct srv_req *r);
int srv_container_files_list(struct srv_req *r);
int srv_container_files_get(struct srv_req *r);
int srv_container_files_delete(struct srv_req *r);
int srv_container_files_content(struct srv_req *r);
int srv_containers_start(struct srv *s);
void srv_containers_stop(void);
/* assistants.c, runs.c: the Assistants API (assistants.h) */
int srv_assistants_create(struct srv_req *r);
int srv_assistants_list(struct srv_req *r);
int srv_assistants_get(struct srv_req *r);
int srv_assistants_modify(struct srv_req *r);
int srv_assistants_delete(struct srv_req *r);
int srv_threads_create(struct srv_req *r);
int srv_threads_get(struct srv_req *r);
int srv_threads_modify(struct srv_req *r);
int srv_threads_delete(struct srv_req *r);
int srv_messages_create(struct srv_req *r);
int srv_messages_list(struct srv_req *r);
int srv_messages_get(struct srv_req *r);
int srv_messages_modify(struct srv_req *r);
int srv_messages_delete(struct srv_req *r);
int srv_runs_create(struct srv_req *r);
int srv_runs_create_thread(struct srv_req *r);
int srv_runs_list(struct srv_req *r);
int srv_runs_get(struct srv_req *r);
int srv_runs_modify(struct srv_req *r);
int srv_runs_cancel(struct srv_req *r);
int srv_runs_submit(struct srv_req *r);
int srv_run_steps_list(struct srv_req *r);
int srv_run_steps_get(struct srv_req *r);
/* vectors_api.c: the Vector stores API */
int srv_vs_create(struct srv_req *r);
int srv_vs_list(struct srv_req *r);
int srv_vs_get(struct srv_req *r);
int srv_vs_modify(struct srv_req *r);
int srv_vs_delete(struct srv_req *r);
int srv_vs_file_create(struct srv_req *r);
int srv_vs_files_list(struct srv_req *r);
int srv_vs_file_get(struct srv_req *r);
int srv_vs_file_delete(struct srv_req *r);
int srv_vs_file_update(struct srv_req *r);
int srv_vs_file_content(struct srv_req *r);
int srv_vs_search(struct srv_req *r);
int srv_vs_batch_create(struct srv_req *r);
int srv_vs_batch_get(struct srv_req *r);
int srv_vs_batch_cancel(struct srv_req *r);
int srv_vs_batch_files(struct srv_req *r);
/* A file attached to store vs (cs, attrs: the request's JSON or NULL): 0,
-1 refused (why says), -2 the disk. */
struct yyjson_val;
int srv_vs_attach(struct srv *srv, const char *vs, const char *file_id,
struct yyjson_val *cs, struct yyjson_val *attrs,
const char **why);
/* A page of the files of store vs (ids not NULL: of these alone). */
int srv_vs_files_page(struct srv_req *r, const char *vs, const char *const *ids,
size_t n_ids);
/* responses.c, conversations.c: OpenAI's Responses API */
int srv_responses_create(struct srv_req *r);
int srv_responses_get(struct srv_req *r);
int srv_responses_delete(struct srv_req *r);
int srv_responses_cancel(struct srv_req *r);
int srv_responses_input_items(struct srv_req *r);
int srv_responses_input_tokens(struct srv_req *r);
int srv_responses_compact(struct srv_req *r);
/* Waits for the responses in the background to be kept, at the end. */
void srv_responses_shutdown(void);
/* At start, the responses a killed server left unfinished are marked as
failed: how many. */
int srv_responses_recover(struct srv_store *st);
int srv_conv_create(struct srv_req *r);
int srv_conv_get(struct srv_req *r);
int srv_conv_update(struct srv_req *r);
int srv_conv_delete(struct srv_req *r);
int srv_conv_items_list(struct srv_req *r);
int srv_conv_items_create(struct srv_req *r);
int srv_conv_item_get(struct srv_req *r);
int srv_conv_item_delete(struct srv_req *r);
/* The value of a query argument of the request, or NULL. */
const char *srv_query(struct srv_req *r, const char *key);
/* A request read (request.c): what it asks of the answer, or why it
cannot be served. */
struct srv_parsed {
int stream, include_usage, echo;
int logprobs; /* chat: true or false; completions: how many */
int store; /* keep the completion (GET /chat/completions) */
char *metadata; /* its metadata, as JSON (NULL: none) */
char *messages; /* the request's messages, as JSON, when stored */
int failed; /* read_...: the error is in the srv_perr */
int ret;
};
void srv_parsed_free(struct srv_parsed *pr);
struct srv_perr {
const char *code; /* OpenAI's error code, for a 400 */
char msg[512];
int oom; /* not the request's fault: out of memory */
};
/* The body of a request to /chat/completions (chat) or /completions into
the job; 0, or -1 with e filled. */
int srv_parse_request(const char *body, size_t n, int chat, struct srv_job *j,
struct srv_parsed *pr, struct srv_perr *e);
/* A new id with a prefix ("chatcmpl-", "cmpl-"), into buf (>= 48 bytes). */
void srv_new_id(char *buf, size_t cap, const char *prefix);
#endif