Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

18.0 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
/*
 * model_tune.c - the threads and GPU each kind of pass runs with, and the
 * profile of the experts a machine uses (see model.h).
 */
#define _GNU_SOURCE
#include "model.h"

#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>

#include "attention.h"
#include "common/pool.h"
#include "common/sysinfo.h"
#include "deltanet.h"
#include "common/devices.h"
#include "tuner.h"
#include "gpu.h"
#include "matvec.h"
#include "quant.h"
#include "vmath.h"

#include "model_impl.h"

/* a mask's combinations without the NPU: what they are in the passes it
   does not take */
static int without_npu(int mask)
{
    int out = 0;
    for (int c = 0; c < JANAS_DEV_COMBOS; c++)
        if (mask >> c & 1)
            out |= 1 << (c & ~JANAS_DEV_NPU);
    return out;
}

/*
 * The user's devices for each kind of pass (--prompt-on, --drafts-on,
 * --reply-on; common/devices.h), in on[] in the order of
 * janas_devices_vars, as far as this machine has them and the pass can use
 * them. The NPU takes blocks only (JANAS_TUNE_BLOCK tokens and more): for
 * the other passes a combination with it is the rest of it, said where it
 * was asked for those passes alone. Prompts kept off the GPU keep its fast
 * prompt off them too; replies asked of it give it single tokens, which an
 * integrated one does not take by default. A device that is not here: the
 * engine chooses, and says why.
 */
static void user_devices(struct janas_llm_model *m, int on[])
{
    static const char *const what[JANAS_DEVICES_OPTIONS] = {"prompts", "drafts",
                                                            "replies"};
    int gpu_here = m->gpu && !m->gpu_off,
        have = 1 | (gpu_here ? 1 << JANAS_DEV_GPU : 0) |
               (m->npu ? 1 << JANAS_DEV_NPU : 0) |
               (gpu_here && m->npu ? 1 << (JANAS_DEV_GPU | JANAS_DEV_NPU) : 0);
    for (int k = 0; k < JANAS_DEVICES_OPTIONS; k++) {
        const char *var = janas_devices_vars[k];
        char why[200], nm[64];
        int mask = janas_devices_mask(getenv(var), why, sizeof why), asked = 0;
        if (mask < 0) {
            fprintf(stderr, "janas: %s: %s; the engine chooses\n", var, why);
            mask = 0;
        }
        for (int c = 0; c < JANAS_DEV_COMBOS; c++)
            asked |= mask >> c & 1 ? c : 0;
        janas_devices_name(mask, nm, sizeof nm);
        if (mask && !(mask & have)) {
            fprintf(stderr,
                    "janas: %s=%s: not on this machine; the engine chooses\n",
                    var, nm);
            if (asked & JANAS_DEV_GPU && !gpu_here)
                fprintf(stderr, "janas:   GPU: %s\n",
                        m->gpu ? "it cannot share the model's memory"
                               : m->gpu_why);
            if (asked & JANAS_DEV_NPU && !m->npu)
                fprintf(stderr, "janas:   NPU: %s\n", m->npu_why);
            mask = 0;
        }
        mask &= mask ? have : 0;
        if (k > 0 && asked & JANAS_DEV_NPU && mask) {
            char rest[64];
            mask = without_npu(mask);
            janas_devices_name(mask, rest, sizeof rest);
            fprintf(stderr,
                    "janas: %s=%s: the NPU takes passes of %d tokens and more "
                    "only; %s on %s\n",
                    var, nm, JANAS_TUNE_BLOCK, what[k], rest);
        }
        on[k] = mask;
    }
    m->prompts_off_gpu =
        on[0] &&
        !(on[0] & (1 << JANAS_DEV_GPU | 1 << (JANAS_DEV_GPU | JANAS_DEV_NPU)));
    if (on[2] >> JANAS_DEV_GPU & 1 && janas_gpu_min_n(m->gpu) > 1)
        janas_gpu_set_min_n(m->gpu, 1);
}

/*
 * The tuner's candidates: every usable core - the performance cores with
 * all their threads and the efficiency cores, not the low-power island,
 * which the layout leaves out - one thread per performance core, and the
 * performance cores' threads alone; each with and without the GPU if there
 * is one. On the development laptop (6 performance cores, 8 efficiency)
 * every core was the fastest or level on six models; on an i9-14900HX (8
 * and 16) every thread replied at 9.9 tokens/s where 16 gave 24.7 (issue
 * #13), and on a 9950X3D one thread per core was 4-15% ahead (#15). So the
 * first passes of each kind try them all and the tuner keeps the threads
 * of the fastest (tuner.h: settled once, never tried again at a longer
 * context, which had moved chats onto fewer cores than they could use).
 * JANAS_TUNE_THREADS=0 keeps every core without trying the others;
 * JANAS_TUNE=0 turns tuning off; JANAS_BLOCK_THREADS fixes the threads for
 * blocks (and turns it off too); JANAS_GPU_ALWAYS=1 makes the GPU variants
 * the defaults for several tokens. The GPU variants are tried for one token
 * too where the GPU takes single tokens (a discrete GPU, or
 * JANAS_GPU_MIN_N=1): the default stays without it.
 */
void janas_m_init_tuner(struct janas_llm_model *m)
{
    struct janas_tune_cand c[JANAS_TUNE_MAX_CANDS];
    int n = 0, opts[3] = {m->all_threads, m->p_cores, m->p_threads};
    const char *bt = getenv("JANAS_BLOCK_THREADS"), *tn = getenv("JANAS_TUNE"),
               *tt = getenv("JANAS_TUNE_THREADS");
    int n_opts = tt && atoi(tt) == 0 ? 1 : 3;
    int block = bt && atoi(bt) > 0 ? atoi(bt) : m->all_threads;
    if (block > m->all_threads)
        block = m->all_threads;
    /* with an NPU every candidate with it too, for blocks only (tuner.c):
       the defaults stay without, unless the user asks for it */
    for (int npu = 0; npu <= (m->npu ? 1 : 0); npu++)
        for (int gpu = 0; gpu <= (m->gpu && !m->gpu_off ? 1 : 0); gpu++)
            for (int i = 0; i < n_opts; i++) {
                int dup = 0;
                for (int k = 0; k < n; k++)
                    dup |= c[k].threads == opts[i] && c[k].gpu == gpu &&
                           c[k].npu == npu;
                if (!dup && n < JANAS_TUNE_MAX_CANDS)
                    c[n++] = (struct janas_tune_cand){
                        .threads = opts[i], .gpu = gpu, .npu = npu};
            }
    int on[JANAS_DEVICES_OPTIONS];
    user_devices(m, on);
    memcpy(m->dev_on, on, sizeof m->dev_on);
    int def[JANAS_TUNE_CLASSES] = {0, 0, 0, 0};
    for (int k = 0; k < n && !c[k].npu; k++) {
        if (!c[k].gpu && c[k].threads == m->all_threads)
            def[0] = k;
        if (!c[k].gpu && c[k].threads == block)
            def[1] = def[2] = def[3] = k;
    }
    /* JANAS_GPU_ALWAYS=1: the GPU variant as the default for passes over
       several tokens (tests, and with JANAS_TUNE=0 a fixed GPU setup) */
    /* with the fast prompt the GPU computes nearly all of a block, and the
       CPU's threads mostly draw the power it would use (the laptop's PL4):
       blocks start on the performance cores' threads, not on all of them -
       of 12 models the fastest blocks were on 6 or 12 threads 11 times,
       Qwen3.5-2B 318 tokens/s on 6 and 309 on 12 against 272 on 20 (10 Oct
       2026) */
    /* the same with the NPU asked for blocks: it computes nearly all of
       one too (Qwen3-4B in whole layers, the threads spinning while they
       waited: 270.6 tokens/s on 6 threads, 262.0 on 12, 252.1 on 20; once
       they slept, 325.5 on 6 and 320.4 on 12; 20 Oct 2026) */
    int fast = m->gpu && !m->gpu_off && janas_gpu_fast_prompt() && !bt &&
               !m->prompts_off_gpu,
        npu_asked =
            on[0] & (1 << JANAS_DEV_NPU | 1 << (JANAS_DEV_GPU | JANAS_DEV_NPU));
    if (fast || (npu_asked && !bt))
        for (int k = 0; k < n; k++)
            if (!c[k].gpu && !c[k].npu && c[k].threads == m->p_threads)
                def[2] = k;
    const char *ga = getenv("JANAS_GPU_ALWAYS");
    if (ga && atoi(ga) > 0)
        for (int k = 0; k < n; k++)
            if (c[k].gpu && !c[k].npu && c[k].threads == block)
                def[1] = def[2] = def[3] = k;
    /* JANAS_NPU_ALWAYS=1: the blocks' default with the NPU (tests: with
       JANAS_TUNE=0 every block then runs with it) */
    const char *na = getenv("JANAS_NPU_ALWAYS");
    if (na && atoi(na) > 0 && m->npu)
        for (int k = 0; k < n; k++)
            if (c[k].npu && c[k].threads == c[def[2]].threads &&
                c[k].gpu == c[def[2]].gpu) {
                def[2] = k;
                break;
            }
    if (bt && block != c[def[1]].threads && n < JANAS_TUNE_MAX_CANDS) {
        c[n] = (struct janas_tune_cand){.threads = block, .gpu = 0};
        def[1] = def[2] = def[3] = n++;
    }
    janas_tuner_init(&m->tuner, c, n, def, (tn && atoi(tn) == 0) || bt);
    janas_tuner_allow_gpu(&m->tuner, 1);
    /* a GPU that takes single tokens (a discrete one, or one the user
       asked for replies) is tried for replies too, and kept only if it
       measures faster */
    int one = m->gpu && !m->gpu_off && janas_gpu_min_n(m->gpu) <= 1;
    janas_tuner_allow_gpu_one(&m->tuner, one);
    /* the user's devices: prompts' blocks and short turns (no NPU there),
       drafts, replies */
    /* the NPU for blocks by itself only where its answers measured at par
       (janas_m_npu_auto); elsewhere when asked */
    const char *nal = getenv("JANAS_NPU_ALWAYS");
    int blocks = on[0];
    if (m->npu && !blocks && !(nal && atoi(nal) > 0) && !janas_m_npu_auto(m))
        blocks = 1 | (m->gpu && !m->gpu_off ? 1 << JANAS_DEV_GPU : 0);
    janas_tuner_allow(&m->tuner, 2, blocks);
    janas_tuner_allow(&m->tuner, 3, without_npu(on[0]));
    janas_tuner_allow(&m->tuner, 1, on[1]);
    janas_tuner_allow(&m->tuner, 0, on[2]);
    /* the saved choices of this machine and model; "1g" in the key when
       the GPU may take replies, so that choices saved without it do not
       stand in for the tries */
    char cpu[128], name[128] = "";
    janas_cpu_name(cpu, sizeof(cpu));
    janas_jns_meta_str(&m->j, "general.name", name, sizeof(name));
    int w =
        snprintf(m->tune_key, sizeof(m->tune_key), "v2|%s|%d/%d/%d|%s%s%s|%s|",
                 cpu, m->p_cores, m->p_threads, m->all_threads,
                 m->gpu && !m->gpu_off ? janas_gpu_name(m->gpu) : "no GPU",
                 one ? " 1g" : "", fast ? " f" : "", name);
    /* the user's devices, when chosen: their own saved choices */
    if ((on[0] || on[1] || on[2]) && w > 0 && (size_t)w < sizeof(m->tune_key))
        w += snprintf(m->tune_key + w, sizeof(m->tune_key) - (size_t)w,
                      "on %x%x%x|", on[0], on[1], on[2]);
    for (int k = 0; k < n && w > 0 && (size_t)w < sizeof(m->tune_key); k++)
        w += snprintf(m->tune_key + w, sizeof(m->tune_key) - (size_t)w,
                      "%d%s%s,", c[k].threads, c[k].gpu ? "g" : "",
                      c[k].npu ? "n" : "");
    janas_tuner_load(&m->tuner, m->tune_key);
}

/* Threads and GPU of a candidate, for the passes that follow, of nv
   vectors. The GPU is kept awake only where it takes such a pass: on an
   integrated GPU a single token's products stay on the CPU
   (janas_gpu_min_n 2) unless the whole token is asked of it
   (JANAS_GPU_TOKEN=1), and keeping it awake there only took the power the
   cores need - a Radeon 780M wrote at 10.5 tokens/s in janas-bench's
   "+ GPU" rows against 17.3 on the CPU alone (issue #16). */
void janas_m_apply_candidate(struct janas_llm_model *m, int cand, uint32_t nv)
{
    /* a borrowed pool belongs to the model that made it: a drafting model
       setting its own thread count would set the target's too */
    if (m->own_compute)
        janas_pool_set_active(m->compute, m->tuner.cand[cand].threads);
    m->gpu_use = m->tuner.cand[cand].gpu;
    if (m->gpu && nv >= JANAS_GPU_FAST_N && janas_gpu_repeatable() &&
        janas_gpu_fast_prompt())
        m->gpu_use = 0; /* the fast products alone: gpu.h */
    m->npu_use = m->tuner.cand[cand].npu;
    janas_m_npu_awake(m, nv);
    if (m->gpu) {
        janas_gpu_keep_awake(m->gpu,
                             m->gpu_use && (nv >= janas_gpu_min_n(m->gpu) ||
                                            janas_m_gpu_token_asked()));
        janas_gpu_set_only_fast(m->gpu, !m->gpu_use);
        janas_gpu_set_pass(m->gpu, nv);
    }
}

void janas_llm_model_set_eco(struct janas_llm_model *m, int eco)
{
    janas_tuner_allow_gpu(&m->tuner, !eco);
}

void janas_llm_model_tuning(const struct janas_llm_model *m, char *buf,
                            size_t len)
{
    janas_tuner_report(&m->tuner, buf, len);
}

int janas_llm_model_threads(const struct janas_llm_model *m, int cls)
{
    if (cls < 0 || cls >= JANAS_TUNE_CLASSES || m->tuner.n_cand <= 0)
        return 0;
    return m->tuner.cand[janas_tuner_best(&m->tuner, cls)].threads;
}

int janas_llm_model_gpu_prompts(const struct janas_llm_model *m)
{
    return m->gpu && !m->gpu_off && m->tuner.allow_gpu &&
           janas_gpu_fast_prompt() && !m->prompts_off_gpu;
}

int janas_llm_model_devices(const struct janas_llm_model *m, int cls)
{
    /* the user's: the tuner's own limits (janas_m_npu_auto) apart */
    return cls == 2   ? m->dev_on[0]
           : cls == 3 ? without_npu(m->dev_on[0])
           : cls == 1 ? m->dev_on[1]
           : cls == 0 ? m->dev_on[2]
                      : 0;
}

int janas_llm_model_gpu_used(const struct janas_llm_model *m)
{
    if (!m->gpu || m->gpu_off)
        return 0;
    int used = 0;
    for (int k = 0; k < JANAS_TUNE_CLASSES; k++) {
        int b = janas_tuner_best(&m->tuner, k);
        if (m->tuner.cand[b].gpu)
            used |= 1 << k;
    }
    return used;
}

int janas_llm_model_npu_used(const struct janas_llm_model *m)
{
    int used = 0;
    for (int k = 0; m->npu && k < JANAS_TUNE_CLASSES; k++)
        if (m->tuner.cand[janas_tuner_best(&m->tuner, k)].npu)
            used |= 1 << k;
    return used;
}

int janas_llm_model_candidates(const struct janas_llm_model *m, int *threads,
                               int *gpu, int *npu, int max)
{
    int n = m->tuner.n_cand < max ? m->tuner.n_cand : max;
    for (int i = 0; i < n; i++) {
        threads[i] = m->tuner.cand[i].threads;
        gpu[i] = m->tuner.cand[i].gpu;
        if (npu)
            npu[i] = m->tuner.cand[i].npu;
    }
    return n;
}

void janas_llm_model_force(struct janas_llm_model *m, int cand)
{
    janas_tuner_force(&m->tuner, cand);
}

/*
 * The experts this machine has used most with this model, counted across
 * sessions in ~/.cache/janas. Without them the first long reply discovers
 * its experts one token at a time: measured on 20 Sep 2026, the second reply
 * of a chat read 14 GB and waited 3.3 seconds for the disk, while every
 * reply after it read 3 GB and waited half a second. The file is small (four
 * bytes per expert of the model) and its counts are added to, so the profile
 * follows how the machine is used.
 */
struct expert_profile {
    char magic[8]; /* JNSXPRT1 */
    uint32_t n_layer, n_expert;
    uint64_t model; /* name, geometry and size of the model file */
    uint64_t tokens;
};

static uint64_t model_fingerprint(const struct janas_llm_model *m)
{
    char name[128] = "";
    janas_jns_meta_str(&m->j, "general.name", name, sizeof(name));
    uint64_t h = janas_fnv1a(name, strlen(name), JANAS_FNV_INIT);
    uint64_t v[3] = {m->j.h.n_layer, m->j.h.n_expert, m->j.h.file_bytes};
    return janas_fnv1a(v, sizeof(v), h);
}

static int profile_path(const struct janas_llm_model *m, char *path, size_t len,
                        int create)
{
    if (janas_cache_dir(path, len, create) != 0)
        return -1;
    size_t n = strlen(path);
    snprintf(path + n, len - n, "/experts-%016llx.bin",
             (unsigned long long)model_fingerprint(m));
    return 0;
}

int janas_llm_model_cache_file(const struct janas_llm_model *m,
                               const char *what, char *path, size_t len,
                               int create)
{
    if (janas_cache_dir(path, len, create) != 0)
        return -1;
    size_t n = strlen(path);
    snprintf(path + n, len - n, "/%s-%016llx.bin", what,
             (unsigned long long)model_fingerprint(m));
    return 0;
}

/* The saved counts, or NULL; *used gets how many experts have a count. */
uint32_t *janas_m_profile_read(const struct janas_llm_model *m, size_t *used)
{
    char path[1024];
    if (profile_path(m, path, sizeof(path), 0) != 0)
        return NULL;
    FILE *f = fopen(path, "rb");
    if (!f)
        return NULL;
    struct expert_profile h;
    size_t n_keys = (size_t)m->j.h.n_layer * m->j.h.n_expert;
    uint32_t *c = NULL;
    if (fread(&h, sizeof(h), 1, f) == 1 &&
        memcmp(h.magic, "JNSXPRT1", 8) == 0 && h.n_layer == m->j.h.n_layer &&
        h.n_expert == m->j.h.n_expert && h.model == model_fingerprint(m) &&
        (c = calloc(n_keys, sizeof(*c))) != NULL &&
        fread(c, sizeof(*c), n_keys, f) != n_keys) {
        free(c);
        c = NULL;
    }
    fclose(f);
    if (c && used)
        for (size_t k = 0; k < n_keys; k++)
            *used += c[k] != 0;
    return c;
}

/* This session's counts added to the file's, halved if they grow large. */
void janas_m_profile_write(const struct janas_llm_model *m)
{
    size_t n_keys = (size_t)m->j.h.n_layer * m->j.h.n_expert;
    uint32_t *c = janas_m_profile_read(m, NULL);
    if (!c && !(c = calloc(n_keys, sizeof(*c))))
        return;
    uint32_t mx = 0;
    for (size_t k = 0; k < n_keys; k++) {
        uint64_t v = (uint64_t)c[k] + m->use_count[k];
        c[k] = v > 0xfffffffful ? 0xfffffffful : (uint32_t)v;
        if (c[k] > mx)
            mx = c[k];
    }
    /* old sessions must not freeze the profile: halve when counts grow */
    if (mx > (1u << 28))
        for (size_t k = 0; k < n_keys; k++)
            c[k] /= 2;
    char path[1024], tmp[1100];
    if (profile_path(m, path, sizeof(path), 1) == 0) {
        snprintf(tmp, sizeof(tmp), "%s.new", path);
        FILE *f = fopen(tmp, "wb");
        if (f) {
            struct expert_profile h = {
                .magic = {'J', 'N', 'S', 'X', 'P', 'R', 'T', '1'},
                .n_layer = m->j.h.n_layer,
                .n_expert = m->j.h.n_expert,
                .model = model_fingerprint(m),
                .tokens = 0};
            int ok = fwrite(&h, sizeof(h), 1, f) == 1 &&
                     fwrite(c, sizeof(*c), n_keys, f) == n_keys;
            ok = fclose(f) == 0 && ok;
            if (ok)
                rename(tmp, path);
            else
                remove(tmp);
        }
    }
    free(c);
}