Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

8.5 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
/*
 * matvec.c - quantized matrix-vector product on a thread pool (see matvec.h).
 */
#include "matvec.h"
#include "common/sysinfo.h"

#include <stdatomic.h>
#include <stdlib.h>

/*
 * Rows are claimed in chunks: large enough that each thread streams a long
 * contiguous run (the hardware prefetcher restarts at every jump: 16-row
 * chunks reached 44 GB/s where 256-row chunks reach 52, on 6 P-cores), small
 * enough that every thread gets about four chunks on small matrices.
 *
 * The chunk is not fixed: it is a fraction of what is left. On cores of three
 * different speeds a fixed chunk leaves everyone waiting for the last one
 * claimed - with four chunks a thread that is half the tail of the run. Taking
 * 1/(2*threads) of the remaining rows keeps the first chunks long, where the
 * prefetcher earns its keep, and ends with chunks of CHUNK_MIN, where nobody
 * waits long. Chunks of different sizes still partition the rows exactly: a
 * fetch_add publishes its own size, so [old, old+chunk) is nobody else's.
 */
#define CHUNK_MIN 16
#define CHUNK_MAX 256

/* Claims the next chunk of [0, total): returns its first row, or total when
   there is nothing left, and writes the end in *end. */
static inline size_t claim(atomic_size_t *next, size_t total, size_t div,
                           size_t *end)
{
    *end = total;
    size_t seen = atomic_load_explicit(next, memory_order_relaxed);
    if (seen >= total)
        return total;
    size_t rem = total - seen, c = rem / div;
    if (c < CHUNK_MIN)
        c = CHUNK_MIN;
    else if (c > CHUNK_MAX)
        c = CHUNK_MAX;
    size_t g0 = atomic_fetch_add_explicit(next, c, memory_order_relaxed);
    if (g0 >= total)
        return total;
    *end = g0 + c < total ? g0 + c : total;
    return g0;
}

/*
 * A chunk's rows on many vectors are taken in pieces whose weights stay in
 * the level-2 cache while every group of eight vectors goes over them: the
 * kernels run the groups outside and the rows inside, so a chunk larger
 * than the cache was read from memory again for each group of eight - the
 * whole matrix once every eight tokens of a prompt. A piece is a quarter of
 * the cache (the rest for the vectors), a cache of 256 KB where the
 * machine does not say. Not by default: on the laptop's Core Ultra (2 MB of
 * L2) a chunk already fits and pieces of 128 KB cost 6% of a prompt (Qwen3-
 * 4B, 52.9 against 56.1 tokens/s, AVX2), and on a Ryzen 5 5500 (512 KB, a
 * prompt 2.2 times slower than llama.cpp's, issue #45) it is not measured
 * yet. JANAS_ROW_PIECE_KB=<n> takes pieces of n KB, -1 a quarter of the
 * L2, 0 or unset the chunk whole.
 */
static size_t row_piece_bytes(void)
{
    static _Atomic size_t bytes; /* 0: not yet known */
    size_t b = atomic_load_explicit(&bytes, memory_order_relaxed);
    if (b)
        return b;
    const char *e = getenv("JANAS_ROW_PIECE_KB");
    if (e && atoi(e) < 0) {
        size_t l2 = janas_l2_bytes();
        b = (l2 ? l2 : (size_t)256 << 10) / 4;
    } else {
        b = e && atoi(e) > 0 ? (size_t)atoi(e) << 10 : SIZE_MAX;
    }
    atomic_store_explicit(&bytes, b, memory_order_relaxed);
    return b;
}

struct matvec_job {
    const struct janas_block_q4k *w;
    const struct janas_block_q8k *x;
    float *y;
    size_t rows;
    size_t nb;  /* blocks per row */
    size_t div; /* the chunk is 1/div of the rows still to claim */
    atomic_size_t next;
};

static void matvec_worker(void *arg, int tid, int n_threads)
{
    (void)tid;
    (void)n_threads;
    struct matvec_job *job = arg;
    for (;;) {
        size_t r1;
        size_t r0 = claim(&job->next, job->rows, job->div, &r1);
        if (r0 >= job->rows)
            break;
        for (size_t r = r0; r < r1; r++)
            job->y[r] = janas_q4k_dot(job->w + r * job->nb, job->x, job->nb);
    }
}

void janas_matvec_q4k(struct janas_pool *pool, const struct janas_block_q4k *w,
                      size_t rows, size_t cols, const struct janas_block_q8k *x,
                      float *y)
{
    struct matvec_job job = {
        .w = w, .x = x, .y = y, .rows = rows, .nb = cols / JANAS_QK};
    job.div = (size_t)janas_pool_active(pool) * 2;
    atomic_init(&job.next, 0);
    janas_pool_run(pool, matvec_worker, &job);
}

struct group_job {
    const struct janas_matvec_task *tasks;
    size_t n;
    size_t *first_row; /* first global row of each task, n + 1 entries */
    size_t div;        /* the chunk is 1/div of the rows still to claim */
    atomic_size_t next;
};

static void group_worker(void *arg, int tid, int n_threads)
{
    (void)tid;
    (void)n_threads;
    struct group_job *job = arg;
    size_t total = job->first_row[job->n];
    size_t t = 0;
    for (;;) {
        size_t g1;
        size_t g0 = claim(&job->next, total, job->div, &g1);
        if (g0 >= total)
            break;
        /* a thread's claims grow, whatever the size: the task index only
           goes forward. A claim may span tasks: each part with its task's
           sizes worked out once, and on one vector all its rows in one
           call (janas_dot_rows) */
        while (g0 < g1) {
            while (job->first_row[t + 1] <= g0)
                t++;
            const struct janas_matvec_task *k = &job->tasks[t];
            size_t r0 = g0 - job->first_row[t];
            size_t end =
                job->first_row[t + 1] < g1 ? job->first_row[t + 1] : g1;
            size_t nr = end - g0;
            /* a row may end inside a group of 256 (janas_dot_rows_n) */
            size_t nb = k->cols / JANAS_QK, tail = k->cols % JANAS_QK;
            size_t row_bytes = janas_qtype_row_bytes(k->type, k->cols);
            const char *w0 = (const char *)k->w + r0 * row_bytes;
            size_t nv = k->n_vec ? k->n_vec : 1;
            size_t xs = k->x_stride ? k->x_stride : nb + (tail != 0);
            size_t ys = k->y_stride ? k->y_stride : k->rows;
            if (k->type == JANAS_Q6_K_P) {
                size_t pb = nb * JANAS_Q6KP_PLANE;
                /* eight rows at a time as tiles, where they serve */
                const uint8_t *pl1 = k->plane[0], *pl0 = k->plane[1];
                size_t r8 = janas_q6k_tile_rows(
                    k->type, w0, row_bytes, pl1 ? pl1 + r0 * pb : NULL,
                    pl0 ? pl0 + r0 * pb : NULL, pb, k->x, xs, nv, nb, nr,
                    k->y + r0, ys);
                for (size_t r = r0 + r8; r < r0 + nr; r++) {
                    const void *row = (const char *)k->w + r * row_bytes;
                    const uint8_t *p1 =
                        k->plane[0] ? (const uint8_t *)k->plane[0] + r * pb
                                    : NULL;
                    const uint8_t *p0 =
                        k->plane[1] ? (const uint8_t *)k->plane[1] + r * pb
                                    : NULL;
                    janas_q6kp_dot_multi(row, p1, p0, k->x, xs, nv, nb,
                                         k->y + r, ys);
                }
            } else if (tail && nv == 1) {
                janas_dot_rows_n(k->type, w0, row_bytes, k->x, k->cols, nr,
                                 k->y + r0);
            } else if (tail) {
                janas_dot_multi_rows_n(k->type, w0, row_bytes, k->x, xs, nv,
                                       k->cols, nr, k->y + r0, ys);
            } else if (nv == 1) {
                janas_dot_rows(k->type, w0, row_bytes, k->x, nb, nr, k->y + r0);
            } else {
                size_t piece = row_piece_bytes() / row_bytes;
                if (nv <= 8 || piece < 1)
                    piece = nv <= 8 ? nr : 1;
                for (size_t i = 0; i < nr; i += piece)
                    janas_dot_multi_rows(
                        k->type, w0 + i * row_bytes, row_bytes, k->x, xs, nv,
                        nb, nr - i < piece ? nr - i : piece, k->y + r0 + i, ys);
            }
            g0 = end;
        }
    }
}

void janas_matvec_q4k_group(struct janas_pool *pool,
                            const struct janas_matvec_task *tasks, size_t n)
{
    size_t first_row_small[65];
    size_t *first_row = first_row_small;
    if (n + 1 > sizeof(first_row_small) / sizeof(first_row_small[0])) {
        first_row = malloc((n + 1) * sizeof(size_t));
        if (!first_row)
            abort();
    }
    first_row[0] = 0;
    for (size_t i = 0; i < n; i++)
        first_row[i + 1] = first_row[i] + tasks[i].rows;

    struct group_job job = {.tasks = tasks, .n = n, .first_row = first_row};
    job.div = (size_t)janas_pool_active(pool) * 2;
    atomic_init(&job.next, 0);
    janas_pool_run(pool, group_worker, &job);
    if (first_row != first_row_small)
        free(first_row);
}