Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

Release 2026-09-27: the prediction file found beside the model

camauri9601e590e33b
diff --git a/ChangeLog.md b/ChangeLog.md
index 3ec7e7c..dbfb69e 100644
--- a/ChangeLog.md
+++ b/ChangeLog.md
@@ -2,6 +2,18 @@
> Curated, user-facing summary of completed work, newest first.
+## [2026-09-27] - The prediction file found beside the model
+
+- **`janas-chat` and `janas-server` find the model's multi-token
+ prediction file by themselves**: of the `.jns` files beside the model,
+ the one whose header fits it - an MTP block of the same architecture,
+ width and experts, or Gemma 4's assistant as wide as the model's output
+ and with its vocabulary - the longest shared start of the name settling
+ a tie. Files over 2 GB (models) are not even opened. `--mtp` still names
+ one, and `--no-mtp` goes without. For programs,
+ `janas_llm_find_mtp()` (and its FreeBASIC declaration); opening a model
+ never looks by itself.
+
## [2026-09-27] - Finer activations for weights in blocks of 32
- **Weights stored in blocks of 32 (Q8_0, Q4_0, Q4_1, Q5_1, IQ4_NL) now
diff --git a/README.md b/README.md
index 0cfa518..024daaf 100644
--- a/README.md
+++ b/README.md
@@ -347,12 +347,17 @@ here in 1.5 GB of memory, E4B in 3.1 GB.
**And chat:**
```sh
-bin/x86_64-linux/janas-chat models/qwen3-next.jns \
- --mtp models/qwen3-next-mtp.jns --stats
-bin/x86_64-linux/janas-chat models/gemma-4-e4b.jns \
- --mtp models/gemma-4-e4b-mtp.jns --stats
+bin/x86_64-linux/janas-chat models/qwen3-next.jns --stats
+bin/x86_64-linux/janas-chat models/gemma-4-e4b.jns --stats
```
+The chat finds the multi-token prediction file by itself: of the `.jns`
+files beside the model, the one whose header fits it (an MTP block of the
+same architecture and width, or Gemma 4's assistant for a model that wide
+and with its vocabulary), and says which it took. `--mtp <file>` names
+one; `--no-mtp` goes without. `janas-server` does the same, and programs
+can ask for it with `janas_llm_find_mtp()`.
+
The first start reads the model's resident weights (a couple of GB) and then
fills the expert cache in the background, while you read and type. The second
reply of a session is the one that shows the machine's real speed.
@@ -368,7 +373,8 @@ janas-chat <model.jns> [options]
| Option | What it does |
|---|---|
-| `--mtp <file>` | the model's multi-token prediction block: faster replies, same text |
+| `--mtp <file>` | the model's multi-token prediction file (an MTP block, or Gemma 4's assistant): faster replies, same text; without it, the one beside the model that fits it |
+| `--no-mtp` | no prediction file, not even one found beside the model |
| `--draft <file>` | a small model of the same family to guess the next tokens: faster replies, same text (see below) |
| `--ctx <tokens>` | context length (default 16384, never more than the model was trained for) |
| `--cache <GiB>` | memory for streamed experts (default: what the machine can spare) |
diff --git a/include/janas/llm.bi b/include/janas/llm.bi
index 710bdc3..4f4bd5a 100644
--- a/include/janas/llm.bi
+++ b/include/janas/llm.bi
@@ -169,6 +169,11 @@ declare function janas_llm_open(byval path as const zstring ptr, _
byval p as const janas_llm_params ptr, _
byval out as janas_llm ptr ptr) as long
declare sub janas_llm_close(byval llm as janas_llm ptr)
+'' The model's multi-token prediction file beside it (see llm.h), its path
+'' in out (cap bytes); JANAS_LLM_EOPEN when there is none.
+declare function janas_llm_find_mtp(byval model_path as const zstring ptr, _
+ byval out as zstring ptr, _
+ byval cap as long) as long
'' Text out of the library: at most cap bytes, NUL-terminated, with *len the
'' full length (JANAS_LLM_ESMALL when it did not fit).
diff --git a/include/janas/llm.h b/include/janas/llm.h
index c2243bc..ec522b8 100644
--- a/include/janas/llm.h
+++ b/include/janas/llm.h
@@ -130,6 +130,18 @@ JANAS_LLM_API int32_t janas_llm_open(const char *path,
janas_llm **out);
JANAS_LLM_API void janas_llm_close(janas_llm *llm);
+/*
+ * The model's multi-token prediction file, looked for beside it: a .jns in
+ * the same directory that fits the model by its header - Gemma 4's
+ * assistant, or an MTP block made by hf2jns_mtp - the one sharing the
+ * longest start with the model's name if several do. Writes its path (at
+ * most cap bytes, NUL-terminated) for janas_llm_params.mtp_path; the model
+ * is not opened. JANAS_LLM_EOPEN when there is none. janas_llm_open never
+ * looks by itself: the caller decides.
+ */
+JANAS_LLM_API int32_t janas_llm_find_mtp(const char *model_path, char *out,
+ int32_t cap);
+
/* The model's name from its metadata ("Qwen3 Next 80B A3B Instruct"; empty
if the file has none). Writes at most cap bytes, NUL-terminated; *len gets
the full length. */
diff --git a/src/chat/main.c b/src/chat/main.c
index 1d60523..98737d3 100644
--- a/src/chat/main.c
+++ b/src/chat/main.c
@@ -199,7 +199,10 @@ static void usage(void)
fprintf(
stderr,
"usage: janas-chat <model.jns> [options]\n"
- " --mtp <file> multi-token prediction block (faster replies)\n"
+ " --mtp <file> multi-token prediction file: an MTP block or\n"
+ " Gemma 4's assistant (faster replies); without\n"
+ " it, the one beside the model that fits it\n"
+ " --no-mtp none, not even one found beside the model\n"
" --draft <file> a small model of the same family to guess the\n"
" next tokens; worth it on a dense model, where\n"
" several can be checked in one pass for little\n"
@@ -591,6 +594,8 @@ struct chat_opts { /* where the words land: main keeps them, this names them */
int *stats, *markdown, *gpu;
const char **mcp_config;
int *mcp, *mcp_auto, *mcp_instructions;
+ int *no_mtp;
+ const char *mtp_found; /* the MTP file found, not asked for: unsaved */
};
static int take_options(int n, char **w, struct chat_opts *o)
@@ -621,6 +626,10 @@ static int take_options(int n, char **w, struct chat_opts *o)
o->cp->recap = 0;
continue;
}
+ if (strcmp(a, "--no-mtp") == 0) {
+ *o->no_mtp = 1;
+ continue;
+ }
if (strcmp(a, "--no-mcp") == 0) {
*o->mcp = 0;
continue;
@@ -804,7 +813,7 @@ static void conf_write(const char *path, const struct chat_opts *o)
fprintf(f, "# janas-chat: written by /save-config. The words are those of\n"
"# the command line, one to a line; the command line wins over\n"
"# this file, and /load-config reads it again.\n");
- if (m->mtp_path)
+ if (m->mtp_path && m->mtp_path != o->mtp_found)
fprintf(f, "--mtp %s\n", m->mtp_path);
fprintf(f, "--ctx %u\n", m->n_ctx);
if (m->cache_bytes)
@@ -1104,10 +1113,12 @@ int main(int argc, char **argv)
int markdown = 1; /* the marks read, not printed; never on a pipe */
int gpu = 1; /* the GPU may be given work, if there is one */
const char *mcp_config = NULL; /* NULL: the default file */
- int mcp = 1, mcp_auto = 0, mcp_instructions = 0;
+ int mcp = 1, mcp_auto = 0, mcp_instructions = 0, no_mtp = 0;
+ static char mtp_found[4096];
struct chat_opts o = {
- &mp, &cp, &system_arg, &stats, &markdown,
- &gpu, &mcp_config, &mcp, &mcp_auto, &mcp_instructions};
+ &mp, &cp, &system_arg, &stats, &markdown,
+ &gpu, &mcp_config, &mcp, &mcp_auto, &mcp_instructions,
+ &no_mtp, mtp_found};
/* the file first, the command line after it: what is typed wins */
char cpath[1024];
char *conf_text = NULL;
@@ -1132,6 +1143,18 @@ int main(int argc, char **argv)
scroll_init(&conv);
scroll_attach(&term, &conv); /* from here the conversation is kept */
int tty = term.tty;
+ /* the model's MTP block or assistant, when not named: the one beside
+ it that fits (--no-mtp: none) */
+ if (!mp.mtp_path && !no_mtp &&
+ janas_llm_find_mtp(argv[1], mtp_found, sizeof(mtp_found)) ==
+ JANAS_LLM_OK) {
+ mp.mtp_path = mtp_found;
+ const char *b = strrchr(mtp_found, '/');
+ fprintf(stderr,
+ "multi-token prediction from %s, found beside the model "
+ "(--no-mtp: without)\n",
+ b ? b + 1 : mtp_found);
+ }
fprintf(stderr, "loading %s ...\n", argv[1]);
janas_llm *llm = NULL;
janas_llm_chat *c = NULL; /* only written when the open succeeded, which
diff --git a/src/llm/api_find.c b/src/llm/api_find.c
new file mode 100644
index 0000000..8510316
--- /dev/null
+++ b/src/llm/api_find.c
@@ -0,0 +1,123 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later
+ Copyright (C) 2026 Maurizio Cammalleri */
+/*
+ * api_find.c - the multi-token prediction file of a model, found beside it
+ * (janas_llm_find_mtp in llm.h).
+ *
+ * A file fits by what its header says, not by its name: Gemma 4's
+ * assistant (gemma4-assistant) whose output is as wide as the model and
+ * whose vocabulary is the model's, or an MTP block (tools/hf2jns_mtp) of
+ * the model's architecture, one layer, as wide, with as many experts. The
+ * name only settles a tie: the file sharing the longest start with the
+ * model's name wins, so that gemma-4-e4b.jns takes gemma-4-e4b-mtp.jns
+ * over another E4B assistant beside it.
+ */
+#define _GNU_SOURCE
+#include <dirent.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+
+#include "llm/chat.h"
+#include "jns.h"
+
+/* no MTP block or assistant is near this size; the models beside one are
+ larger, and their headers are not worth reading */
+#define MTP_MAX_BYTES ((uint64_t)2 << 30)
+
+struct model_shape {
+ char arch[32];
+ uint32_t d_model, n_expert;
+ uint64_t n_vocab;
+};
+
+static uint64_t vocab_of(const struct janas_jns *j)
+{
+ const struct janas_jns_tensor *t = janas_jns_tensor(j, "token_embd.weight");
+ return t ? t->dims[1] : 0;
+}
+
+static int fits(const struct janas_jns *j, const struct model_shape *m)
+{
+ const struct janas_jns_header *h = &j->h;
+ if (strncmp(h->arch, "gemma4-assistant", sizeof(h->arch)) == 0) {
+ uint32_t out = 0;
+ return strcmp(m->arch, "gemma4") == 0 &&
+ janas_jns_meta_u32(j, "gemma4-assistant.embedding_length_out",
+ &out) == 0 &&
+ out == m->d_model && vocab_of(j) == m->n_vocab;
+ }
+ return strncmp(h->arch, m->arch, sizeof(h->arch)) == 0 && h->n_layer == 1 &&
+ h->d_model == m->d_model && h->n_expert == m->n_expert &&
+ janas_jns_tensor(j, "mtp.eh_proj.weight") != NULL;
+}
+
+static size_t common_start(const char *a, const char *b)
+{
+ size_t n = 0;
+ while (a[n] && a[n] == b[n])
+ n++;
+ return n;
+}
+
+int32_t janas_llm_find_mtp(const char *model_path, char *out, int32_t cap)
+{
+ if (!model_path || !out || cap < 2)
+ return janas_api_fail(JANAS_LLM_EINVAL, "no path or no room");
+ struct janas_jns j;
+ char err[256];
+ if (janas_jns_open(model_path, &j, err, sizeof(err)) != 0)
+ return janas_api_fail(JANAS_LLM_EOPEN, "%s: %s", model_path, err);
+ struct model_shape m = {.d_model = j.h.d_model,
+ .n_expert = j.h.n_expert,
+ .n_vocab = vocab_of(&j)};
+ snprintf(m.arch, sizeof(m.arch), "%.31s", j.h.arch);
+ janas_jns_close(&j);
+
+ /* the model's directory and its own name */
+ char dir[4096];
+ snprintf(dir, sizeof(dir), "%s", model_path);
+ char *slash = strrchr(dir, '/');
+ const char *base = slash ? model_path + (slash - dir) + 1 : model_path;
+ if (slash)
+ *(slash == dir ? slash + 1 : slash) = 0;
+ else
+ snprintf(dir, sizeof(dir), ".");
+ DIR *d = opendir(dir);
+ if (!d)
+ return janas_api_fail(JANAS_LLM_EOPEN, "cannot read %s", dir);
+ char best[8192] = "";
+ size_t best_score = 0;
+ struct dirent *e;
+ while ((e = readdir(d))) {
+ size_t n = strlen(e->d_name);
+ if (n < 5 || strcmp(e->d_name + n - 4, ".jns") != 0 ||
+ strcmp(e->d_name, base) == 0)
+ continue;
+ char path[8192];
+ snprintf(path, sizeof(path), "%s/%s", dir, e->d_name);
+ struct stat st;
+ if (stat(path, &st) != 0 || !S_ISREG(st.st_mode) ||
+ (uint64_t)st.st_size > MTP_MAX_BYTES)
+ continue;
+ if (janas_jns_open(path, &j, err, sizeof(err)) != 0)
+ continue;
+ int ok = fits(&j, &m);
+ janas_jns_close(&j);
+ size_t score = common_start(e->d_name, base) + 1;
+ if (ok && (score > best_score ||
+ (score == best_score && strcmp(path, best) < 0))) {
+ best_score = score;
+ snprintf(best, sizeof(best), "%s", path);
+ }
+ }
+ closedir(d);
+ if (!best[0])
+ return janas_api_fail(JANAS_LLM_EOPEN, "no MTP file beside %s",
+ model_path);
+ if (strlen(best) + 1 > (size_t)cap)
+ return janas_api_fail(JANAS_LLM_ESMALL, "path longer than %d", cap);
+ memcpy(out, best, strlen(best) + 1);
+ return JANAS_LLM_OK;
+}
diff --git a/src/llm/model_ple.c b/src/llm/model_ple.c
index 1e44bfb..fec7739 100644
--- a/src/llm/model_ple.c
+++ b/src/llm/model_ple.c
@@ -15,6 +15,7 @@
*/
#define _GNU_SOURCE
#include <math.h>
+#include <stdatomic.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
@@ -35,7 +36,13 @@ static float round_bf16(float f)
return janas_u2f(u);
}
+/* rows claimed a few at a time from a shared counter, as the quantized
+ products do: equal shares left the performance cores waiting for the
+ efficiency cores */
+#define PLE_CHUNK 16
+
struct bf16_job {
+ atomic_uint next;
const uint16_t *w;
const float *x; /* n rows of cols, rounded to bf16 */
float *y; /* n rows of rows */
@@ -66,8 +73,45 @@ dot_bf16_avx2(const uint16_t *w, const float *x, uint32_t n)
r += janas_u2f((uint32_t)w[i] << 16) * x[i];
return r;
}
+
+/* four tokens against one row, the row converted once: for each token the
+ same accumulator and the same order as dot_bf16_avx2, so the same bits */
+__attribute__((target("avx2,fma"))) static void
+dot4_bf16_avx2(const uint16_t *w, const float *const x[4], uint32_t n,
+ float out[4])
+{
+ __m256 acc[4] = {_mm256_setzero_ps(), _mm256_setzero_ps(),
+ _mm256_setzero_ps(), _mm256_setzero_ps()};
+ uint32_t i = 0;
+ for (; i + 8 <= n; i += 8) {
+ __m128i h = _mm_loadu_si128((const __m128i *)(const void *)(w + i));
+ __m256 wf = _mm256_castsi256_ps(
+ _mm256_slli_epi32(_mm256_cvtepu16_epi32(h), 16));
+ for (int t = 0; t < 4; t++)
+ acc[t] = _mm256_fmadd_ps(wf, _mm256_loadu_ps(x[t] + i), acc[t]);
+ }
+ for (int t = 0; t < 4; t++) {
+ __m128 s = _mm_add_ps(_mm256_castps256_ps128(acc[t]),
+ _mm256_extractf128_ps(acc[t], 1));
+ s = _mm_add_ps(s, _mm_movehl_ps(s, s));
+ s = _mm_add_ss(s, _mm_movehdup_ps(s));
+ float r = _mm_cvtss_f32(s);
+ for (uint32_t k = i; k < n; k++)
+ r += janas_u2f((uint32_t)w[k] << 16) * x[t][k];
+ out[t] = r;
+ }
+}
#endif
+static int have_avx2(void)
+{
+#if defined(__x86_64__)
+ return __builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma");
+#else
+ return 0;
+#endif
+}
+
static float dot_bf16(const uint16_t *w, const float *x, uint32_t n)
{
#if defined(__x86_64__)
@@ -83,19 +127,38 @@ static float dot_bf16(const uint16_t *w, const float *x, uint32_t n)
/* rows split in equal runs, each row read once for all the tokens */
static void bf16_worker(void *arg, int tid, int n_threads)
{
- const struct bf16_job *b = arg;
- uint32_t per = (b->rows + (uint32_t)n_threads - 1) / (uint32_t)n_threads;
- uint32_t r0 = (uint32_t)tid * per;
- uint32_t r1 = r0 + per < b->rows ? r0 + per : b->rows;
- for (uint32_t r = r0; r < r1; r++) {
- const uint16_t *w = b->w + (size_t)r * b->cols;
- for (uint32_t j = 0; j < b->n; j++)
- b->y[(size_t)j * b->rows + r] =
- dot_bf16(w, b->x + (size_t)j * b->cols, b->cols) * b->scale;
+ (void)tid;
+ (void)n_threads;
+ struct bf16_job *b = arg;
+ int v = have_avx2();
+ for (;;) {
+ uint32_t r0 = atomic_fetch_add(&b->next, PLE_CHUNK);
+ if (r0 >= b->rows)
+ break;
+ uint32_t r1 = r0 + PLE_CHUNK < b->rows ? r0 + PLE_CHUNK : b->rows;
+ for (uint32_t r = r0; r < r1; r++) {
+ const uint16_t *w = b->w + (size_t)r * b->cols;
+ uint32_t j = 0;
+#if defined(__x86_64__)
+ for (; v && j + 4 <= b->n; j += 4) {
+ const float *x[4];
+ float o[4];
+ for (int t = 0; t < 4; t++)
+ x[t] = b->x + (size_t)(j + t) * b->cols;
+ dot4_bf16_avx2(w, x, b->cols, o);
+ for (int t = 0; t < 4; t++)
+ b->y[(size_t)(j + t) * b->rows + r] = o[t] * b->scale;
+ }
+#endif
+ for (; j < b->n; j++)
+ b->y[(size_t)j * b->rows + r] =
+ dot_bf16(w, b->x + (size_t)j * b->cols, b->cols) * b->scale;
+ }
}
}
struct f32_job {
+ atomic_uint next;
const float *w, *x;
float *y;
uint32_t rows, cols, n;
@@ -118,6 +181,29 @@ dot_f32_avx2(const float *w, const float *x, uint32_t n)
r += w[i] * x[i];
return r;
}
+
+/* four tokens against one row, as dot_f32_avx2 for each: the same bits */
+__attribute__((target("avx2,fma"))) static void
+dot4_f32_avx2(const float *w, const float *const x[4], uint32_t n, float out[4])
+{
+ __m256 a0[4], a1[4];
+ for (int t = 0; t < 4; t++)
+ a0[t] = a1[t] = _mm256_setzero_ps();
+ uint32_t i = 0;
+ for (; i + 16 <= n; i += 16) {
+ __m256 w0 = _mm256_loadu_ps(w + i), w1 = _mm256_loadu_ps(w + i + 8);
+ for (int t = 0; t < 4; t++) {
+ a0[t] = _mm256_fmadd_ps(w0, _mm256_loadu_ps(x[t] + i), a0[t]);
+ a1[t] = _mm256_fmadd_ps(w1, _mm256_loadu_ps(x[t] + i + 8), a1[t]);
+ }
+ }
+ for (int t = 0; t < 4; t++) {
+ float r = janas_hsum8(_mm256_add_ps(a0[t], a1[t]));
+ for (uint32_t k = i; k < n; k++)
+ r += w[k] * x[t][k];
+ out[t] = r;
+ }
+}
#endif
/* one dot product, the same for a row and a token whatever the pass */
@@ -137,15 +223,33 @@ static float dot_f32(const float *w, const float *x, uint32_t n)
matrices of E2B's and E4B's branches are 2.6 MB a layer */
static void f32_worker(void *arg, int tid, int n_threads)
{
- const struct f32_job *b = arg;
- uint32_t per = (b->rows + (uint32_t)n_threads - 1) / (uint32_t)n_threads;
- uint32_t r0 = (uint32_t)tid * per;
- uint32_t r1 = r0 + per < b->rows ? r0 + per : b->rows;
- for (uint32_t r = r0; r < r1; r++) {
- const float *w = b->w + (size_t)r * b->cols;
- for (uint32_t j = 0; j < b->n; j++)
- b->y[(size_t)j * b->rows + r] =
- dot_f32(w, b->x + (size_t)j * b->cols, b->cols);
+ (void)tid;
+ (void)n_threads;
+ struct f32_job *b = arg;
+ int v = have_avx2();
+ for (;;) {
+ uint32_t r0 = atomic_fetch_add(&b->next, PLE_CHUNK);
+ if (r0 >= b->rows)
+ break;
+ uint32_t r1 = r0 + PLE_CHUNK < b->rows ? r0 + PLE_CHUNK : b->rows;
+ for (uint32_t r = r0; r < r1; r++) {
+ const float *w = b->w + (size_t)r * b->cols;
+ uint32_t j = 0;
+#if defined(__x86_64__)
+ for (; v && j + 4 <= b->n; j += 4) {
+ const float *x[4];
+ float o[4];
+ for (int t = 0; t < 4; t++)
+ x[t] = b->x + (size_t)(j + t) * b->cols;
+ dot4_f32_avx2(w, x, b->cols, o);
+ for (int t = 0; t < 4; t++)
+ b->y[(size_t)(j + t) * b->rows + r] = o[t];
+ }
+#endif
+ for (; j < b->n; j++)
+ b->y[(size_t)j * b->rows + r] =
+ dot_f32(w, b->x + (size_t)j * b->cols, b->cols);
+ }
}
}
@@ -159,6 +263,7 @@ static void product(struct janas_llm_model *m, const struct janas_jns_tensor *t,
if (t->type == 0) {
struct f32_job b = {
.w = f32(m, t), .x = x, .y = y, .rows = rows, .cols = cols, .n = n};
+ atomic_init(&b.next, 0);
janas_pool_run(m->compute, f32_worker, &b);
return;
}
@@ -195,6 +300,7 @@ int janas_m_ple_inputs(struct janas_llm_model *m, const int32_t *tokens,
.cols = dm,
.n = n,
.scale = 1.0f / sqrtf((float)dm)};
+ atomic_init(&b.next, 0);
if (n < 2)
bf16_worker(&b, 0, 1);
else
diff --git a/src/server/engine.c b/src/server/engine.c
index cc4b62f..e665049 100644
--- a/src/server/engine.c
+++ b/src/server/engine.c
@@ -626,7 +626,18 @@ struct srv_engine *srv_engine_start(const struct srv_config *cfg, char *err,
e->warn_input = cfg->warn_input;
pthread_mutex_init(&e->mu, NULL);
pthread_cond_init(&e->cv, NULL);
- if (janas_llm_open(cfg->model_path, &cfg->llm, &e->llm) != JANAS_LLM_OK ||
+ /* the model's MTP block or assistant, when not named: the one beside
+ it that fits */
+ static char mtp_found[4096];
+ struct janas_llm_params lp = cfg->llm;
+ if (!lp.mtp_path && !cfg->no_mtp &&
+ janas_llm_find_mtp(cfg->model_path, mtp_found, sizeof(mtp_found)) ==
+ JANAS_LLM_OK) {
+ lp.mtp_path = mtp_found;
+ fprintf(stderr, "janas-server: multi-token prediction from %s\n",
+ mtp_found);
+ }
+ if (janas_llm_open(cfg->model_path, &lp, &e->llm) != JANAS_LLM_OK ||
janas_llm_chat_create(e->llm, NULL, &e->chat) != JANAS_LLM_OK) {
snprintf(err, err_len, "%s", janas_llm_last_error());
janas_llm_close(e->llm);
diff --git a/src/server/main.c b/src/server/main.c
index 7272ef6..4e073fe 100644
--- a/src/server/main.c
+++ b/src/server/main.c
@@ -36,6 +36,9 @@ static void usage(FILE *f)
" --reserve GIB memory left to other programs when the cache\n"
" is automatic (default: a fifth of the machine's)\n"
" --mtp FILE the model's prediction block, for speed\n"
+ " (without it, the one beside the model that\n"
+ " fits it: an MTP block or Gemma 4's assistant)\n"
+ " --no-mtp none, not even one found beside the model\n"
" --draft FILE a small model of the same family, for speed\n"
" --mode M auto, eco or max (default auto)\n"
" --no-gpu never give the GPU work\n"
@@ -167,6 +170,8 @@ int main(int argc, char **argv)
c->thinking = strcmp(v, "off") ? 1 : 0;
} else if (!strcmp(a, "--embedding-model")) {
c->embed_path = arg(argc, argv, &i);
+ } else if (!strcmp(a, "--no-mtp")) {
+ c->no_mtp = 1;
} else if (!strcmp(a, "--no-gpu")) {
c->no_gpu = 1;
} else if (!strcmp(a, "--keep")) {
diff --git a/src/server/server.h b/src/server/server.h
index 2301b22..0b5cc42 100644
--- a/src/server/server.h
+++ b/src/server/server.h
@@ -27,6 +27,7 @@ struct srv_config {
const char *model_path;
struct janas_llm_params llm; /* how the model is opened */
int no_gpu;
+ int no_mtp; /* no MTP file, not even one found beside the model */
const char *model_id; /* the name clients use; default: the file name */
const char *host; /* address to listen on (default 127.0.0.1) */
uint16_t port; /* default 8080 */