37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
/*
* model.c - open() and the Model class: a model file opened on an engine
* thread of its own, its chat, how its preparations are going, close().
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "node/node_impl.h"
#define GIB 1073741824.0
struct open_job {
char *path;
struct janas_llm_params p;
char *mtp, *draft;
int find_mtp; /* mtp: true, the block beside the model if there is one */
int no_gpu;
char found[4096]; /* the block found */
/* what the model says of itself, for the Model's properties */
char *name, *description, *system;
int reasons;
};
static void open_drop(void *data)
{
struct open_job *oj = data;
if (!oj)
return;
free(oj->path);
free(oj->mtp);
free(oj->draft);
free(oj->name);
free(oj->description);
free(oj->system);
free(oj);
}
static const char *const open_known[] = {
"context", "cacheGiB", "reserveGiB", "mtp", "draft",
"mode", "gpu", "preload", "expertBits", "headBits",
"attention", "quality", NULL};
/* one of the strings, as its index; -1 after throwing */
static int choice(napi_env env, napi_value o, const char *name,
const char *const *names, int *out)
{
char *s = NULL;
int r = janas_node_opt_string(env, o, "open", name, &s);
if (r <= 0)
return r;
for (int i = 0; names[i]; i++)
if (!strcmp(names[i], s)) {
free(s);
*out = i;
return 1;
}
janas_node_type_error(env, "janas: open: %s cannot be '%s'", name, s);
free(s);
return -1;
}
static int open_options(napi_env env, napi_value o, struct open_job *oj)
{
const char *w = "open";
static const char *const modes[] = {"auto", "eco", "max", NULL};
static const char *const attn[] = {"auto", "exact", "fast", NULL};
static const char *const quality[] = {"low", "medium", "high", NULL};
int64_t i;
double d;
int b, r, k;
if (janas_node_options(env, o, w, open_known) < 0)
return -1;
if ((r = janas_node_opt_int(env, o, w, "context", 1, UINT32_MAX, &i)) < 0)
return -1;
if (r)
oj->p.n_ctx = (uint32_t)i;
if ((r = janas_node_opt_number(env, o, w, "cacheGiB", 0, 1 << 20, &d)) < 0)
return -1;
if (r)
oj->p.cache_bytes = (uint64_t)(d * GIB);
if ((r = janas_node_opt_number(env, o, w, "reserveGiB", 0, 1 << 20, &d)) <
0)
return -1;
if (r)
oj->p.reserve_bytes = (uint64_t)(d * GIB);
napi_value v;
napi_valuetype t;
if (o && napi_typeof(env, o, &t) == napi_ok && t == napi_object &&
napi_get_named_property(env, o, "mtp", &v) == napi_ok &&
napi_typeof(env, v, &t) == napi_ok && t != napi_undefined) {
if (t == napi_boolean) {
bool on = false;
napi_get_value_bool(env, v, &on);
oj->find_mtp = on;
} else if (janas_node_opt_string(env, o, w, "mtp", &oj->mtp) < 0)
return -1;
}
if (janas_node_opt_string(env, o, w, "draft", &oj->draft) < 0)
return -1;
if ((r = choice(env, o, "mode", modes, &k)) < 0)
return -1;
if (r)
oj->p.mode = k == 1 ? JANAS_LLM_MODE_ECO
: k == 2 ? JANAS_LLM_MODE_MAX
: JANAS_LLM_MODE_AUTO;
if ((r = janas_node_opt_bool(env, o, w, "gpu", &b)) < 0)
return -1;
if (r)
oj->no_gpu = !b;
if ((r = janas_node_opt_bool(env, o, w, "preload", &b)) < 0)
return -1;
if (r)
oj->p.no_preload = !b;
if ((r = janas_node_opt_int(env, o, w, "expertBits", 2, 6, &i)) < 0)
return -1;
if (r && i != 2 && i != 4 && i != 6) {
janas_node_type_error(env, "janas: open: expertBits is 2, 4 or 6");
return -1;
}
if (r)
oj->p.expert_bits = (int32_t)i;
if ((r = janas_node_opt_int(env, o, w, "headBits", 4, 6, &i)) < 0)
return -1;
if (r && i == 5) {
janas_node_type_error(env, "janas: open: headBits is 4 or 6");
return -1;
}
if (r)
oj->p.head_bits = (int32_t)i;
if ((r = choice(env, o, "attention", attn, &k)) < 0)
return -1;
if (r)
oj->p.attn_scores = k == 1 ? JANAS_LLM_ATTN_EXACT
: k == 2 ? JANAS_LLM_ATTN_FAST
: JANAS_LLM_ATTN_AUTO;
if ((r = choice(env, o, "quality", quality, &k)) < 0)
return -1;
if (r)
oj->p.min_quality = k; /* JANAS_LLM_QUALITY_*: low 0, high 2 */
return 0;
}
/* a text the library writes (janas_llm_name and the like), malloc'd */
static char *take(int32_t (*get)(const janas_llm *, char *, int32_t, int32_t *),
const janas_llm *llm)
{
char small[256];
int32_t len = 0;
if (get(llm, small, sizeof(small), &len) != JANAS_LLM_OK || len < 0)
return strdup("");
if (len < (int32_t)sizeof(small))
return strdup(small);
char *s = malloc((size_t)len + 1);
if (s && get(llm, s, len + 1, &len) != JANAS_LLM_OK)
s[0] = 0;
return s;
}
static void open_run(struct janas_node_job *j)
{
struct janas_node_model *m = j->m;
struct open_job *oj = j->data;
if (oj->find_mtp && janas_llm_find_mtp(oj->path, oj->found,
sizeof(oj->found)) == JANAS_LLM_OK)
oj->p.mtp_path = oj->found;
else if (oj->mtp)
oj->p.mtp_path = oj->mtp;
oj->p.draft_path = oj->draft;
j->rc = janas_llm_open(oj->path, &oj->p, &m->llm);
if (j->rc != JANAS_LLM_OK) {
snprintf(j->error, sizeof(j->error), "%s", janas_llm_last_error());
m->llm = NULL;
j->last = 1; /* no model: the thread ends */
return;
}
/* where there is no GPU there is nothing to keep it off */
if (oj->no_gpu)
janas_llm_set_gpu(m->llm, 0);
oj->name = take(janas_llm_name, m->llm);
oj->description = take(janas_llm_describe, m->llm);
oj->system = take(janas_llm_default_system, m->llm);
oj->reasons = janas_llm_reasons(m->llm);
}
static void model_finalize(napi_env env, void *data, void *hint)
{
(void)hint;
janas_node_release(env, data);
}
static napi_value text_or_empty(napi_env env, const char *s)
{
return janas_node_string(env, s ? s : "", JANAS_NAPI_AUTO_LENGTH);
}
static napi_value open_value(napi_env env, struct janas_node_job *j)
{
struct janas_node_model *m = j->m;
struct open_job *oj = j->data;
napi_value obj = janas_node_new(env, m->st->model_class,
&janas_node_model_tag, m, model_finalize);
if (!obj)
return NULL;
janas_node_hold(m);
napi_property_descriptor d[] = {
{.utf8name = "path",
.value = text_or_empty(env, oj->path),
.attributes = napi_enumerable},
{.utf8name = "name",
.value = text_or_empty(env, oj->name),
.attributes = napi_enumerable},
{.utf8name = "description",
.value = text_or_empty(env, oj->description),
.attributes = napi_enumerable},
{.utf8name = "defaultSystem",
.value = text_or_empty(env, oj->system),
.attributes = napi_enumerable},
{.utf8name = "reasons",
.value = janas_node_bool(env, oj->reasons),
.attributes = napi_enumerable},
{.utf8name = "mtp",
.value = oj->p.mtp_path ? text_or_empty(env, oj->p.mtp_path)
: janas_node_bool(env, 0),
.attributes = napi_enumerable},
};
napi_define_properties(env, obj, sizeof(d) / sizeof(d[0]), d);
return obj;
}
static void open_after(napi_env env, struct janas_node_job *j)
{
struct janas_node_model *m = j->m;
if (j->rc < 0)
m->closing = 1; /* the thread has ended: back as the last job */
else if (!m->holders)
janas_node_close(env, m, NULL); /* its object could not be made */
}
napi_value janas_node_open(napi_env env, napi_callback_info info)
{
size_t argc = 2;
napi_value argv[2] = {NULL, NULL};
napi_get_cb_info(env, info, &argc, argv, NULL, NULL);
struct janas_node_env *st = janas_node_env(env);
size_t len = 0;
char *path = argc ? janas_node_text(env, argv[0], &len) : NULL;
if (!st || !path)
return len == SIZE_MAX
? janas_node_throw(env, JANAS_LLM_ENOMEM,
"janas: out of memory")
: janas_node_type_error(
env, "janas: open(path, options): the path of a "
".jns file, as a string");
struct open_job *oj = calloc(1, sizeof(*oj));
if (!oj) {
free(path);
return janas_node_throw(env, JANAS_LLM_ENOMEM, "janas: out of memory");
}
oj->path = path;
janas_llm_params_default(&oj->p);
if (open_options(env, argc > 1 ? argv[1] : NULL, oj) < 0) {
open_drop(oj);
return NULL;
}
struct janas_node_model *m = janas_node_engine(env, st);
struct janas_node_job *j = m ? janas_node_job(m, open_run) : NULL;
napi_value p = j ? janas_node_promise(env, &j->deferred) : NULL;
if (!p) {
open_drop(oj);
free(j);
if (m) /* the thread ends with the close it is given */
janas_node_close(env, m, NULL);
return janas_node_throw(env, JANAS_LLM_ENOMEM,
"janas: open: no memory or no thread for "
"the engine");
}
j->data = oj;
j->drop = open_drop;
j->value = open_value;
j->after = open_after;
janas_node_post(env, j);
return p;
}
static napi_value model_chat(napi_env env, napi_callback_info info)
{
size_t argc = 1;
napi_value argv[1] = {NULL};
struct janas_node_model *m = janas_node_self(
env, info, &janas_node_model_tag, &argc, argv, "Model.chat");
if (!m)
return NULL;
return janas_node_chat(env, m, argc ? argv[0] : NULL);
}
static napi_value model_close(napi_env env, napi_callback_info info)
{
struct janas_node_model *m = janas_node_self(
env, info, &janas_node_model_tag, NULL, NULL, "Model.close");
if (!m)
return NULL;
napi_deferred d;
napi_value p = janas_node_promise(env, &d);
if (p)
janas_node_close(env, m, d);
return p;
}
/* How the model's preparations are going: the expert cache filling from
what this machine used most, the NPU made ready. The library allows both
questions while the engine's thread works. */
static napi_value model_status(napi_env env, napi_callback_info info)
{
struct janas_node_model *m = janas_node_self(
env, info, &janas_node_model_tag, NULL, NULL, "Model.status");
if (!m)
return NULL;
int open = m->engine && !m->closing && m->llm;
int64_t done = 0, total = 0;
int32_t ndone = 0, ntotal = 0;
double seconds = 0;
int preparing = 0;
if (open) {
janas_llm_preload(m->llm, &done, &total);
preparing = janas_llm_npu_preparing(m->llm, &ndone, &ntotal, &seconds);
}
napi_value o, pre, npu;
if (napi_create_object(env, &o) != napi_ok ||
napi_create_object(env, &pre) != napi_ok ||
napi_create_object(env, &npu) != napi_ok)
return NULL;
janas_node_put(env, pre, "done", janas_node_number(env, (double)done));
janas_node_put(env, pre, "total", janas_node_number(env, (double)total));
janas_node_put(env, npu, "preparing", janas_node_bool(env, preparing > 0));
janas_node_put(env, npu, "done", janas_node_number(env, ndone));
janas_node_put(env, npu, "total", janas_node_number(env, ntotal));
janas_node_put(env, npu, "seconds", janas_node_number(env, seconds));
janas_node_put(env, o, "open", janas_node_bool(env, open));
janas_node_put(env, o, "busy", janas_node_bool(env, m->pending > 0));
janas_node_put(env, o, "preload", pre);
janas_node_put(env, o, "npu", npu);
return o;
}
napi_value janas_node_model_class(napi_env env)
{
const napi_property_attributes method = napi_writable | napi_configurable;
napi_property_descriptor d[] = {
{.utf8name = "chat", .method = model_chat, .attributes = method},
{.utf8name = "status", .method = model_status, .attributes = method},
{.utf8name = "close", .method = model_close, .attributes = method},
};
napi_value cls = NULL;
if (napi_define_class(env, "Model", JANAS_NAPI_AUTO_LENGTH,
janas_node_constructor,
(void *)"a Model comes from open()",
sizeof(d) / sizeof(d[0]), d, &cls) != napi_ok)
return NULL;
return cls;
}