37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
/*
* janas-nsm - a word's explication in the Natural Semantic Metalanguage
* (NSM), written by a model trained for it (DeepNSM, arXiv 2505.11764:
* Llama 3 with a LoRA adapter, converted with tools/hf2gguf), and what the
* explication is made of: its primes, function words and other words
* ("molecules"), DeepNSM's legality, whether it is circular (score.h).
*
* janas-nsm <model.jns> <word> [example sentence...] one word
* janas-nsm <model.jns> --batch <in.jsonl> a file of words
* janas-nsm --score <in.jsonl> explications given
*
* A word's lines in a batch are JSON objects with "word" and "examples"
* (DeepNSM's dataset as it is); --score reads "word" and "explication".
* A batch writes one JSON line per explication (--out, else the standard
* output) and a summary on the standard error.
*
* The prompt is the model card's ("Word: ...\nExamples:\n...\nParaphrase:");
* --prompt paper is the one the paper's numbers were obtained with (its
* evaluation script's, indentation included). The sampling is DeepNSM's
* (temperature 0.7, top-k 50, top-p 0.95, at most 512 tokens).
*/
#include <errno.h>
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "janas/llm.h"
#include "llm/json.h"
#include "score.h"
#include "vocab.h"
struct opts {
const char *model, *batch, *score, *out, *prompt, *molecules;
int strict;
enum nsm_lex lex;
int samples, max, quiet;
uint64_t seed;
float temp, top_p;
int top_k;
};
static void usage(void)
{
fprintf(stderr,
"usage: janas-nsm <model.jns> <word> [example sentence...] "
"[options]\n"
" janas-nsm <model.jns> --batch <in.jsonl> [options]\n"
" janas-nsm --score <in.jsonl> [options]\n"
"A word's explication in NSM primes, by a model trained for it\n"
"(DeepNSM), and what it is made of: primes, function words and\n"
"other words (molecules), legality 10 (primes - molecules) / "
"words,\n"
"circularity (the word itself in its explication).\n"
" --batch <file> JSON lines with \"word\" and \"examples\"\n"
" --score <file> JSON lines with \"word\" and \"explication\": "
"count\n"
" them, no model\n"
" --out <file> a batch's JSON lines (default: standard "
"output)\n"
" -n <count> explications a word (default 1)\n"
" --seed <n> the sampling's seed (default 1)\n"
" --prompt <card|paper> the model card's prompt (default) or "
"the\n"
" one the paper's numbers were obtained with\n"
" --lexicon <paper|full> primes as DeepNSM's scores counted "
"them\n"
" (default), or the 65 with their forms and "
"phrases\n"
" --strict the model may write only the primes' forms, "
"function\n"
" words, punctuation and the molecules: its "
"output head\n"
" computes those tokens alone\n"
" --molecules <file> the molecules --strict allows, one word a "
"line\n"
" (default: a list of everyday words; an empty "
"file: none)\n"
" --temp <t> --top-k <k> --top-p <p> --max <tokens>\n"
" sampling (default 0.7, 50, 0.95, 512)\n"
" --quiet no progress on the standard error\n");
}
static char *read_file(const char *path, size_t *len)
{
FILE *f = fopen(path, "rb");
if (!f)
return NULL;
fseek(f, 0, SEEK_END);
long n = ftell(f);
fseek(f, 0, SEEK_SET);
char *b = n >= 0 ? malloc((size_t)n + 1) : NULL;
if (b && fread(b, 1, (size_t)n, f) != (size_t)n) {
free(b);
b = NULL;
}
fclose(f);
if (b) {
b[n] = 0;
*len = (size_t)n;
}
return b;
}
/* --- the summary of a batch --- */
struct tally {
size_t n;
double leg, leg2, primes, molecules, circular;
double out_tokens, out_s, in_tokens, in_s;
};
static void tally_add(struct tally *t, const struct nsm_score *s)
{
t->n++;
t->leg += s->legality;
t->leg2 += s->legality * s->legality;
t->primes += s->primes_ratio;
t->molecules += s->molecules_ratio;
t->circular += s->circular;
}
static void tally_print(const struct tally *t)
{
if (!t->n)
return;
double n = (double)t->n, m = t->leg / n;
double sd = t->n > 1 ? sqrt((t->leg2 - n * m * m) / (n - 1)) : 0;
fprintf(stderr,
"%zu explications: legality %.2f +- %.2f (standard error), "
"primes %.1f%%, molecules %.1f%%, circular %.1f%%\n",
t->n, m, sd / sqrt(n), 100 * t->primes / n, 100 * t->molecules / n,
100 * t->circular / n);
if (t->out_s > 0)
fprintf(stderr,
"writing %.0f tokens in %.1f s, %.1f tokens/s; reading the "
"prompts %.0f tokens in %.1f s, %.0f tokens/s\n",
t->out_tokens, t->out_s, t->out_tokens / t->out_s, t->in_tokens,
t->in_s, t->in_s > 0 ? t->in_tokens / t->in_s : 0);
}
static void put_score(struct janas_buf *b, const struct nsm_score *s)
{
janas_buf_printf(b,
", \"legality\": %.4f, \"primes\": %.4f, "
"\"molecules\": %.4f, \"words\": %zu, \"circular\": %s",
s->legality, s->primes_ratio, s->molecules_ratio, s->words,
s->circular ? "true" : "false");
}
/* --- the model --- */
struct gen {
janas_llm *m;
janas_llm_chat *c;
struct janas_llm_chat_params p;
};
/* The words of a file, one a line, NULL-terminated (malloc'd, into *text). */
static const char **read_words(const char *path, char **text)
{
size_t len;
*text = read_file(path, &len);
if (!*text) {
fprintf(stderr, "janas-nsm: %s: %s\n", path, strerror(errno));
exit(1);
}
size_t n = 0; /* lines end in \n, \r or both: as many as either */
for (size_t i = 0; i < len; i++)
n += (*text)[i] == '\n' || (*text)[i] == '\r';
const char **w = calloc(n + 2, sizeof(*w));
size_t k = 0;
for (char *p = strtok(*text, "\r\n"); p && w; p = strtok(NULL, "\r\n"))
if (*p)
w[k++] = p;
return w;
}
/* --strict: the replies' vocabulary */
static void restrict_vocab(struct gen *g, const struct opts *o)
{
char *text = NULL;
const char **mol = o->molecules ? read_words(o->molecules, &text) : NULL;
int32_t *ids = NULL;
long n = nsm_vocab_tokens(g->m, mol, &ids);
if (n <= 0 || janas_llm_chat_vocab(g->c, (int32_t)n, ids) != JANAS_LLM_OK) {
fprintf(stderr, "janas-nsm: --strict: %s\n",
n <= 0 ? "no vocabulary" : janas_llm_last_error());
exit(1);
}
if (!o->quiet)
fprintf(stderr, "janas-nsm: --strict, %ld tokens of the vocabulary\n",
n);
free(ids);
free(mol);
free(text);
}
static void gen_open(struct gen *g, const struct opts *o)
{
struct janas_llm_params lp;
janas_llm_params_default(&lp);
lp.n_ctx = 4096;
if (!o->quiet)
fprintf(stderr, "janas-nsm: loading %s ...\n", o->model);
if (janas_llm_open(o->model, &lp, &g->m) != JANAS_LLM_OK) {
fprintf(stderr, "janas-nsm: %s\n", janas_llm_last_error());
exit(1);
}
janas_llm_chat_params_default(&g->p);
g->p.temperature = o->temp;
g->p.top_k = o->top_k;
g->p.top_p = o->top_p;
g->p.min_p = 0;
g->p.max_reply = o->max;
g->p.speculate = 0;
if (janas_llm_chat_create(g->m, &g->p, &g->c) != JANAS_LLM_OK) {
fprintf(stderr, "janas-nsm: %s\n", janas_llm_last_error());
exit(1);
}
if (o->strict)
restrict_vocab(g, o);
}
/* The prompt for a word and its examples. */
static void build_prompt(struct janas_buf *b, const struct opts *o,
const char *word, const char *const *ex, int n_ex)
{
int paper = !strcmp(o->prompt, "paper");
/* nsm_evaluation.py's f-string keeps its source's indentation */
const char *ind = paper ? " " : "";
janas_buf_printf(b, "Word: %s\n%sExamples:\n%s", word, ind, ind);
for (int i = 0; i < n_ex; i++)
janas_buf_printf(b, "%s%s", i ? "\n" : "", ex[i]);
janas_buf_printf(b, "\n%sParaphrase:%s%s", ind, paper ? "\n" : "", ind);
}
/* One explication into out; its tokens and times into t. */
static int explain(struct gen *g, const char *prompt, size_t n, uint64_t seed,
struct janas_buf *out, struct tally *t)
{
g->p.seed = seed;
if (janas_llm_chat_reset(g->c) != JANAS_LLM_OK ||
janas_llm_chat_set_params(g->c, &g->p) != JANAS_LLM_OK ||
janas_llm_chat_prompt(g->c, prompt, (int32_t)n) != JANAS_LLM_OK)
return -1;
out->n = 0;
char piece[256];
for (;;) {
int32_t len = 0;
int32_t rc = janas_llm_chat_next(g->c, piece, sizeof(piece), &len);
if (rc == JANAS_LLM_DONE)
break;
if (rc != JANAS_LLM_OK)
return -1;
janas_buf_put(out, piece, (size_t)len);
}
struct janas_llm_chat_stats st = {.size = sizeof(st)};
janas_llm_chat_stats(g->c, &st);
t->out_tokens += st.output_tokens;
t->out_s += st.output_seconds;
t->in_tokens += st.input_tokens;
t->in_s += st.input_seconds;
/* the model writes a space after "Paraphrase:", and a newline last */
size_t a = 0;
while (a < out->n && (out->p[a] == ' ' || out->p[a] == '\n'))
a++;
while (out->n > a &&
(out->p[out->n - 1] == ' ' || out->p[out->n - 1] == '\n'))
out->n--;
memmove(out->p, out->p + a, out->n - a);
out->n -= a;
janas_buf_put(out, "", 1);
out->n--;
return 0;
}
/* --- the three ways to run --- */
static int one_word(const struct opts *o, const char *word,
const char *const *ex, int n_ex)
{
struct gen g;
gen_open(&g, o);
struct janas_buf pr = {0}, out = {0};
build_prompt(&pr, o, word, ex, n_ex);
struct tally t = {0};
for (int k = 0; k < o->samples; k++) {
if (explain(&g, pr.p, pr.n, o->seed * 1000003u + (uint64_t)k, &out,
&t) != 0) {
fprintf(stderr, "janas-nsm: %s\n", janas_llm_last_error());
return 1;
}
struct nsm_score s;
nsm_score_text(o->lex, out.p, out.n, word, &s);
tally_add(&t, &s);
printf("%s%.*s\n\n[legality %.2f, primes %.0f%%, molecules %.0f%%, "
"%zu words%s]\n",
k ? "\n" : "", (int)out.n, out.p, s.legality,
100 * s.primes_ratio, 100 * s.molecules_ratio, s.words,
s.circular ? ", circular" : "");
}
if (o->samples > 1 || !o->quiet)
tally_print(&t);
janas_buf_free(&pr);
janas_buf_free(&out);
janas_llm_chat_destroy(g.c);
janas_llm_close(g.m);
return 0;
}
/* Calls f on each JSON line of the file. */
static int each_line(const char *path,
int (*f)(const struct janas_json *, size_t, void *),
void *ctx)
{
size_t len;
char *text = read_file(path, &len);
if (!text) {
fprintf(stderr, "janas-nsm: %s: %s\n", path, strerror(errno));
return 1;
}
size_t line = 0;
for (char *p = text; *p;) {
char *e = strchr(p, '\n');
size_t n = e ? (size_t)(e - p) : strlen(p);
line++;
if (n) {
char err[128];
struct janas_json_doc *d = janas_json_parse(p, n, err, sizeof(err));
if (!d) {
fprintf(stderr, "janas-nsm: %s:%zu: %s\n", path, line, err);
free(text);
return 1;
}
int rc = f(janas_json_root(d), line, ctx);
janas_json_free(d);
if (rc) {
free(text);
return rc;
}
}
p += n + (e != NULL);
}
free(text);
return 0;
}
struct run {
const struct opts *o;
struct gen g;
FILE *out;
struct tally t;
size_t index;
};
static int score_line(const struct janas_json *v, size_t line, void *ctx)
{
struct run *r = ctx;
const char *w = janas_json_str(janas_json_get(v, "word"));
const struct janas_json *e = janas_json_get(v, "explication");
if (!e)
e = janas_json_get(v, "text");
if (!w || !janas_json_str(e)) {
fprintf(stderr,
"janas-nsm: line %zu: no \"word\" and \"explication\"\n", line);
return 1;
}
struct nsm_score s;
nsm_score_text(r->o->lex, e->s, e->n, w, &s);
tally_add(&r->t, &s);
struct janas_buf b = {0};
janas_buf_puts(&b, "{\"word\": ");
janas_json_write_str(&b, w, strlen(w));
put_score(&b, &s);
janas_buf_puts(&b, "}\n");
fwrite(b.p, 1, b.n, r->out);
janas_buf_free(&b);
return 0;
}
static int batch_line(const struct janas_json *v, size_t line, void *ctx)
{
struct run *r = ctx;
const char *w = janas_json_str(janas_json_get(v, "word"));
const struct janas_json *ex = janas_json_get(v, "examples");
if (!w || !ex || ex->type != JANAS_JSON_ARRAY) {
fprintf(stderr, "janas-nsm: line %zu: no \"word\" and \"examples\"\n",
line);
return 1;
}
const char *exs[64];
int n_ex = 0;
for (const struct janas_json *x = ex->child; x && n_ex < 64; x = x->next)
if (janas_json_str(x))
exs[n_ex++] = x->s;
struct janas_buf pr = {0}, out = {0}, b = {0};
build_prompt(&pr, r->o, w, exs, n_ex);
for (int k = 0; k < r->o->samples; k++) {
double s0 = r->t.out_s, t0 = r->t.out_tokens;
uint64_t seed = (r->o->seed * 1000003u + r->index) * 31u + (uint64_t)k;
if (explain(&r->g, pr.p, pr.n, seed, &out, &r->t) != 0) {
fprintf(stderr, "janas-nsm: line %zu: %s\n", line,
janas_llm_last_error());
return 1;
}
struct nsm_score s;
nsm_score_text(r->o->lex, out.p, out.n, w, &s);
tally_add(&r->t, &s);
b.n = 0;
janas_buf_puts(&b, "{\"word\": ");
janas_json_write_str(&b, w, strlen(w));
janas_buf_printf(&b, ", \"sample\": %d, \"explication\": ", k);
janas_json_write_str(&b, out.p, out.n);
put_score(&b, &s);
janas_buf_printf(&b, ", \"tokens\": %.0f, \"seconds\": %.3f}\n",
r->t.out_tokens - t0, r->t.out_s - s0);
fwrite(b.p, 1, b.n, r->out);
fflush(r->out);
}
r->index++;
if (!r->o->quiet && r->index % 10 == 0)
fprintf(stderr, "janas-nsm: %zu words, %.1f tokens/s\n", r->index,
r->t.out_s > 0 ? r->t.out_tokens / r->t.out_s : 0);
janas_buf_free(&pr);
janas_buf_free(&out);
janas_buf_free(&b);
return 0;
}
static int run_file(const struct opts *o, int with_model)
{
struct run r = {.o = o, .out = stdout};
if (o->out && !(r.out = fopen(o->out, "w"))) {
fprintf(stderr, "janas-nsm: %s: %s\n", o->out, strerror(errno));
return 1;
}
if (with_model)
gen_open(&r.g, o);
int rc = each_line(with_model ? o->batch : o->score,
with_model ? batch_line : score_line, &r);
tally_print(&r.t);
if (r.out != stdout)
fclose(r.out);
if (with_model) {
janas_llm_chat_destroy(r.g.c);
janas_llm_close(r.g.m);
}
return rc;
}
int main(int argc, char **argv)
{
struct opts o = {.prompt = "card",
.lex = NSM_LEX_PAPER,
.samples = 1,
.max = 512,
.seed = 1,
.temp = 0.7f,
.top_p = 0.95f,
.top_k = 50};
const char *pos[66];
int n_pos = 0;
for (int i = 1; i < argc; i++) {
const char *a = argv[i];
const char *v = i + 1 < argc ? argv[i + 1] : NULL;
#define TAKES(name) (!strcmp(a, name) && v && ++i)
if (!strcmp(a, "-h") || !strcmp(a, "--help")) {
usage();
return 0;
} else if (TAKES("--batch"))
o.batch = v;
else if (TAKES("--score"))
o.score = v;
else if (TAKES("--out"))
o.out = v;
else if (TAKES("-n"))
o.samples = atoi(v);
else if (TAKES("--seed"))
o.seed = strtoull(v, NULL, 10);
else if (TAKES("--prompt"))
o.prompt = v;
else if (TAKES("--lexicon"))
o.lex = !strcmp(v, "full") ? NSM_LEX_FULL : NSM_LEX_PAPER;
else if (TAKES("--temp"))
o.temp = (float)atof(v);
else if (TAKES("--top-k"))
o.top_k = atoi(v);
else if (TAKES("--top-p"))
o.top_p = (float)atof(v);
else if (TAKES("--max"))
o.max = atoi(v);
else if (TAKES("--molecules"))
o.molecules = v;
else if (!strcmp(a, "--strict"))
o.strict = 1;
else if (!strcmp(a, "--quiet"))
o.quiet = 1;
else if (a[0] == '-' && a[1]) {
fprintf(stderr, "janas-nsm: unknown option %s\n", a);
usage();
return 2;
} else if (n_pos < 66)
pos[n_pos++] = a;
#undef TAKES
}
if (strcmp(o.prompt, "card") && strcmp(o.prompt, "paper")) {
fprintf(stderr, "janas-nsm: --prompt card or paper\n");
return 2;
}
if (o.samples < 1 || o.max < 1) {
usage();
return 2;
}
if (o.score)
return run_file(&o, 0);
if (n_pos < 1) {
usage();
return 2;
}
o.model = pos[0];
if (o.batch)
return run_file(&o, 1);
if (n_pos < 2) {
usage();
return 2;
}
return one_word(&o, pos[1], pos + 2, n_pos - 2);
}