Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

29.4 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
/*
 * model_ana.c - the Ana-Net probe (job/docs/progetto_ana.md; the name is
 * provisional). A fraction of the feed-forward neurons is switched off
 * before the down product, chosen by a score, to measure what each choice
 * costs the answers against a reference. A probe: nothing is saved (the
 * down product still reads every column), and only the CPU's path has it
 * (JANAS_GPU=0).
 *
 * JANAS_ANA_MODE, a pair's neurons (a token through one expert, or through
 * the dense feed-forward) ranked by themselves:
 *   mag   |h|, the known way;
 *   agree how much the neuron's inputs agree: for gate and for up,
 *         |sum w x| / sum |w x| (1 when every contribution pulls the same
 *         way, near 0 when they cancel), the two multiplied;
 *   both  |h| times the agreement;
 *   magw  |h| times the norm of the neuron's column of down: what switching
 *         it off takes from the output;
 *   bothw magw times the square root of the agreement;
 *   random.
 * Or a token's neurons, all its experts' together, ranked by what each
 * gives the output, its expert's weight w (the router's, or the shared
 * expert's gate) in it, so that a light expert loses more neurons than a
 * heavy one:
 *   tokmag  w |h|;  tokw  w |h| times the column's norm;
 *   tokboth tokw times the square root of the agreement.
 * JANAS_ANA_FRAC: the fraction off (0.4).
 * JANAS_ANA_COARSE=1 (the per-token modes): the choice made on h from gate
 * and up read at two bits a weight (Q4_K's nibbles, Q8_0's bytes, cut to
 * their two high bits), the neurons chosen then computed whole - a
 * predictor that reads a third of gate and up; at the end, how many of the
 * neurons the true choice keeps the two-bit one kept too.
 * JANAS_ANA_GULOW=n (the per-token modes): the n routed experts of least
 * weight in each token have their gate and up read at two bits a weight,
 * as JANAS_ANA_COARSE reads them; the shared expert whole.
 * JANAS_ANA_GRADE=f6,f4,f2 (the per-token modes): of a token's neurons
 * ranked, the first f6 keep all of down, the next f4 down at four bits, the
 * next f2 at two (its bit planes, or Q8_0's bytes cut), the rest off.
 * JANAS_ANA_STATS=file (the per-token modes): for every pair of every token
 * a record - its token, layer, expert (n_expert: the shared one), neurons
 * ff and weight w as five 32-bit words (w a float), then ff floats of h
 * before anything is off, then ff bytes, 1 for a neuron kept - for
 * job/tests/py/ana_stats.py.
 * JANAS_ANA_XSTATS=file (the per-token modes): each token's input to the
 * feed-forward of each layer - its token, layer and width as three 32-bit
 * words, then that many floats - once for every pass that ranks it (a
 * layer's experts in two passes write it twice).
 * JANAS_ANA_FRAMES=dir: for each token computed alone, a picture of the
 * network (dir/frame_NNNNN.ppm, a row a layer, a column a neuron of a pair
 * of the token: red its agreement against the pair's highest, green its
 * magnitude against the pair's largest, blue where it was switched off),
 * the first JANAS_ANA_FRAMES_MAX (32) of them.
 */
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "model_impl.h"
#include "quant.h"

enum {
    ANA_MAG = 1,
    ANA_AGREE,
    ANA_BOTH,
    ANA_MAGW,
    ANA_BOTHW,
    ANA_RANDOM,
    ANA_TOKMAG,
    ANA_TOKW,
    ANA_TOKBOTH
};

static const char *const NAMES[] = {"",     "mag",    "agree",  "both",
                                    "magw", "bothw",  "random", "tokmag",
                                    "tokw", "tokboth"};

static int needs_agree(int mode)
{
    return mode == ANA_AGREE || mode == ANA_BOTH || mode == ANA_BOTHW ||
           mode == ANA_TOKBOTH;
}

static int needs_coln(int mode)
{
    return mode == ANA_MAGW || mode == ANA_BOTHW || mode == ANA_TOKW ||
           mode == ANA_TOKBOTH;
}

int janas_m_ana_init(struct janas_llm_model *m, size_t pairs)
{
    const char *e = getenv("JANAS_ANA_MODE");
    m->ana_mode = 0;
    for (int i = 1; e && i < (int)(sizeof(NAMES) / sizeof(NAMES[0])); i++)
        if (!strcmp(e, NAMES[i]))
            m->ana_mode = i;
    m->ana_tok = m->ana_mode >= ANA_TOKMAG;
    e = getenv("JANAS_ANA_FRAC");
    m->ana_frac = e ? (float)atof(e) : 0.4f;
    if (m->ana_frac < 0.0f)
        m->ana_frac = 0.0f;
    if (m->ana_frac > 0.95f)
        m->ana_frac = 0.95f;
    e = getenv("JANAS_ANA_FRAMES");
    if (e && *e)
        snprintf(m->ana_dir, sizeof(m->ana_dir), "%s", e);
    e = getenv("JANAS_ANA_FRAMES_MAX");
    m->ana_frames_max = e ? atoi(e) : 32;
    e = getenv("JANAS_ANA_GRADE");
    if (m->ana_tok && e &&
        sscanf(e, "%f,%f,%f", &m->ana_grade[0], &m->ana_grade[1],
               &m->ana_grade[2]) == 3) {
        m->ana_frac =
            1.0f - (m->ana_grade[0] + m->ana_grade[1] + m->ana_grade[2]);
        if (m->ana_frac < 0.0f)
            m->ana_frac = 0.0f;
        if (!(m->ana_level = calloc(pairs * m->ff_buf, 1)))
            return -1;
    }
    e = getenv("JANAS_ANA_GULOW");
    m->ana_gulow = m->ana_tok && e ? atoi(e) : 0;
    if (m->ana_gulow > 0 && !(m->ana_low = calloc(pairs, 1)))
        return -1;
    if (m->ana_gulow > 0)
        fprintf(stderr,
                "ana: gate and up at two bits for the %d experts of "
                "least weight in a token\n",
                m->ana_gulow);
    e = getenv("JANAS_ANA_COARSE");
    if (m->ana_tok && e && atoi(e) &&
        !(m->ana_coarse = calloc(pairs * m->ff_buf, sizeof(float))))
        return -1;
    e = getenv("JANAS_ANA_STATS");
    if (e && *e && m->ana_tok && !(m->ana_stats = fopen(e, "wb")))
        return -1;
    e = getenv("JANAS_ANA_XSTATS");
    if (e && *e && m->ana_tok && !(m->ana_xstats = fopen(e, "wb")))
        return -1;
    if (!m->ana_mode && !m->ana_dir[0])
        return 0;
    m->ana_exp = calloc(pairs, sizeof(uint32_t));
    if (!m->ana_exp)
        return -1;
    if (needs_agree(m->ana_mode) || m->ana_dir[0]) {
        m->ana_agree = calloc(pairs * m->ff_buf, sizeof(float));
        if (!m->ana_agree)
            return -1;
    }
    if (needs_coln(m->ana_mode)) {
        size_t slots = (size_t)m->n_layer * (m->n_expert + 1);
        m->ana_coln = calloc(slots * m->ff_buf, sizeof(float));
        m->ana_coln_ok = calloc(slots, 1);
        if (!m->ana_coln || !m->ana_coln_ok)
            return -1;
    }
    if (m->ana_dir[0]) {
        /* the routed experts of a token and a shared one, or a dense
           feed-forward */
        m->ana_width = m->ff_buf * (m->n_expert > 1 ? m->k + 1 : 1);
        m->ana_frame = calloc((size_t)m->n_layer * m->ana_width, 3);
        if (!m->ana_frame)
            return -1;
    }
    if (m->ana_level)
        fprintf(stderr,
                "ana: down at 6, 4, 2 bits for %.0f%%, %.0f%%, %.0f%% "
                "of a token's neurons\n",
                100.0 * m->ana_grade[0], 100.0 * m->ana_grade[1],
                100.0 * m->ana_grade[2]);
    if (m->ana_coarse)
        fprintf(stderr, "ana: the choice made on gate and up at two bits\n");
    fprintf(stderr,
            "ana: %s, %.0f%% of each %s's neurons off%s%s (a probe: the CPU's "
            "path only)\n",
            m->ana_mode ? NAMES[m->ana_mode] : "nothing switched off",
            m->ana_mode ? 100.0 * m->ana_frac : 0.0,
            m->ana_tok ? "token" : "pair",
            m->ana_dir[0] ? ", pictures in " : "", m->ana_dir);
    return 0;
}

void janas_m_ana_free(struct janas_llm_model *m)
{
    if (m->ana_true_kept)
        fprintf(stderr,
                "ana: the two-bit choice kept %.1f%% of the neurons the true "
                "one kept\n",
                100.0 * (double)m->ana_both_kept / (double)m->ana_true_kept);
    free(m->ana_coarse);
    free(m->ana_level);
    free(m->ana_low);
    m->ana_coarse = NULL;
    m->ana_level = NULL;
    m->ana_low = NULL;
    free(m->ana_agree);
    free(m->ana_frame);
    free(m->ana_exp);
    free(m->ana_coln);
    free(m->ana_coln_ok);
    if (m->ana_stats)
        fclose(m->ana_stats);
    if (m->ana_xstats)
        fclose(m->ana_xstats);
    m->ana_stats = m->ana_xstats = NULL;
    m->ana_agree = m->ana_coln = NULL;
    m->ana_frame = m->ana_coln_ok = NULL;
    m->ana_exp = NULL;
}

/* A run's agreements: for each neuron (a row of gate and of up), for each of
   its pairs, how much the products w x of the token's inputs agree. The
   rows unpacked once for all the run's pairs; neurons split among the
   threads. */
struct agree_job {
    struct janas_llm_model *m;
    const uint8_t *w[2];
    const struct janas_jns_matrix *mx;
    uint32_t a, b;
};

/* A row of gate or up at two bits a weight: Q4_K's nibbles and Q8_0's bytes
   cut to their two high bits, the middle of what those leave; other types
   as they are. tmp: room for the row's blocks. */
static void coarse_row(int type, const uint8_t *w, float *y, uint32_t n,
                       uint8_t *tmp)
{
    size_t nb = n / JANAS_QK, bs = janas_qtype_block_size(type);
    memcpy(tmp, w, nb * bs);
    if (type == JANAS_Q4_K) {
        for (size_t b = 0; b < nb; b++) {
            struct janas_block_q4k *k =
                (struct janas_block_q4k *)(tmp + b * bs);
            for (int i = 0; i < JANAS_QK / 2; i++)
                k->qs[i] = (uint8_t)(((k->qs[i] & 0x0C) | 0x01) |
                                     ((k->qs[i] & 0xC0) | 0x20));
        }
    } else if (type == JANAS_Q8_0) {
        struct janas_block_q8_0 *k = (struct janas_block_q8_0 *)tmp;
        for (size_t b = 0; b < nb * 8; b++)
            for (int i = 0; i < 32; i++)
                k[b].qs[i] = (int8_t)(((uint8_t)k[b].qs[i] & 0xC0) | 0x20);
    }
    janas_dequantize(type, tmp, y, n);
}

static void agree_worker(void *arg, int tid, int n_threads)
{
    struct agree_job *J = arg;
    struct janas_llm_model *m = J->m;
    uint32_t rows = J->mx[0].rows, cols = J->mx[0].cols, dm = m->d_model;
    size_t rb[2];
    for (int p = 0; p < 2; p++)
        rb[p] = (size_t)J->mx[p].cols / JANAS_QK *
                janas_qtype_block_size((int)J->mx[p].type);
    float *row = malloc(4 * (size_t)cols * sizeof(float));
    uint8_t *tmp = malloc(rb[0] > rb[1] ? rb[0] : rb[1]);
    if (!row || !tmp) {
        free(row);
        free(tmp);
        return;
    }
    for (uint32_t j = (uint32_t)tid; j < rows; j += (uint32_t)n_threads) {
        for (int p = 0; p < 2; p++)
            janas_dequantize((int)J->mx[p].type, J->w[p] + j * rb[p],
                             row + (size_t)p * cols, cols);
        int two = m->ana_coarse != NULL;
        for (uint32_t g = J->a; m->ana_low && !two && g < J->b; g++)
            two = m->ana_low[g];
        for (int p = 0; two && p < 2; p++)
            coarse_row((int)J->mx[p].type, J->w[p] + j * rb[p],
                       row + (size_t)(2 + p) * cols, cols, tmp);
        for (uint32_t g = J->a; two && g < J->b; g++) {
            if (!m->ana_coarse && !m->ana_low[g])
                continue;
            const float *x = m->xn + (size_t)m->pair_tok[m->order[g]] * dm;
            double v[2] = {0, 0};
            for (int p = 0; p < 2; p++)
                for (uint32_t i = 0; i < cols; i++)
                    v[p] += (double)row[(size_t)(2 + p) * cols + i] * x[i];
            if (m->ana_coarse) {
                double gs = v[0] / (1.0 + exp(-v[0]));
                m->ana_coarse[(size_t)g * m->ff_buf + j] = (float)(gs * v[1]);
            }
            if (m->ana_low && m->ana_low[g]) {
                /* this pair's gate and up are the two-bit ones */
                m->gate[(size_t)g * m->ff_buf + j] = (float)v[0];
                m->up[(size_t)g * m->ff_buf + j] = (float)v[1];
            }
        }
        if (!m->ana_agree)
            continue;
        for (uint32_t g = J->a; g < J->b; g++) {
            const float *x = m->xn + (size_t)m->pair_tok[m->order[g]] * dm;
            float c[2];
            for (int p = 0; p < 2; p++) {
                const float *w = row + (size_t)p * cols;
                double s = 0, sa = 0;
                for (uint32_t i = 0; i < cols; i++) {
                    double v = (double)w[i] * x[i];
                    s += v;
                    sa += fabs(v);
                }
                c[p] = sa > 0 ? (float)(fabs(s) / sa) : 0.0f;
            }
            m->ana_agree[(size_t)g * m->ff_buf + j] = c[0] * c[1];
        }
    }
    free(row);
    free(tmp);
}

void janas_m_ana_agree(struct janas_llm_model *m, const uint8_t *wg,
                       const uint8_t *wu, const struct janas_jns_matrix *mx,
                       uint32_t a, uint32_t b)
{
    if ((!m->ana_agree && !m->ana_coarse && !m->ana_low) ||
        mx[0].cols != m->d_model || mx[1].cols != m->d_model ||
        mx[0].rows != mx[1].rows || mx[0].rows > m->ff_buf ||
        !janas_qtype_block_size((int)mx[0].type) ||
        !janas_qtype_block_size((int)mx[1].type))
        return;
    struct agree_job J = {.m = m, .w = {wg, wu}, .mx = mx, .a = a, .b = b};
    janas_pool_run(m->compute, agree_worker, &J);
}

/* The norms of the columns of expert e's down matrix in the current layer
   (rows the model's width, a column a neuron), once: what switching a
   neuron off takes from the output, per unit of |h|. Bit planes unpacked
   with those the cache holds; a type that cannot be unpacked, norms of 1. */
void janas_m_ana_coln(struct janas_llm_model *m, uint32_t e,
                      const uint8_t *down, const uint8_t *p1, const uint8_t *p0,
                      const struct janas_jns_matrix *mx)
{
    size_t slot = (size_t)m->ana_layer * (m->n_expert + 1) + e;
    if (!m->ana_coln || m->ana_coln_ok[slot] || mx->cols > m->ff_buf)
        return;
    float *out = m->ana_coln + slot * m->ff_buf;
    uint32_t rows = mx->rows, cols = mx->cols, nb = cols / JANAS_QK;
    int planes = mx->type == JANAS_Q6_K_P;
    size_t bs = planes ? sizeof(struct janas_block_q6kp)
                       : janas_qtype_block_size((int)mx->type);
    float *y = malloc(cols * sizeof(float));
    double *acc = calloc(cols, sizeof(double));
    if (!y || !acc || !bs) {
        for (uint32_t j = 0; j < cols; j++)
            out[j] = 1.0f;
    } else {
        for (uint32_t r = 0; r < rows; r++) {
            if (planes)
                janas_q6kp_dequantize(
                    (const struct janas_block_q6kp *)(down +
                                                      (size_t)r * nb * bs),
                    p1 ? p1 + (size_t)r * nb * JANAS_Q6KP_PLANE : NULL,
                    p1 && p0 ? p0 + (size_t)r * nb * JANAS_Q6KP_PLANE : NULL, y,
                    cols);
            else
                janas_dequantize((int)mx->type, down + (size_t)r * nb * bs, y,
                                 cols);
            for (uint32_t j = 0; j < cols; j++)
                acc[j] += (double)y[j] * y[j];
        }
        for (uint32_t j = 0; j < cols; j++)
            out[j] = (float)sqrt(acc[j]);
    }
    free(y);
    free(acc);
    m->ana_coln_ok[slot] = 1;
}

/* A fixed pseudo-random number of a neuron of a pair in a layer. */
static float noise(uint32_t layer, size_t g, uint32_t j)
{
    uint64_t z = ((uint64_t)layer << 40) ^ ((uint64_t)g << 20) ^ j;
    z += 0x9E3779B97F4A7C15ull;
    z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull;
    z = (z ^ (z >> 27)) * 0x94D049BB133111EBull;
    z ^= z >> 31;
    return (float)(z >> 40) / (float)(1u << 24);
}

/* Neuron j of pair row g by the mode's score (higher: kept). */
static float score(const struct janas_llm_model *m, size_t g, uint32_t j,
                   float h)
{
    const float *agree = m->ana_agree ? m->ana_agree + g * m->ff_buf : NULL;
    float a = agree ? agree[j] : 1.0f, mag = fabsf(h);
    float col = 1.0f;
    if (m->ana_coln && m->ana_exp) {
        size_t slot = (size_t)m->ana_layer * (m->n_expert + 1) + m->ana_exp[g];
        col = m->ana_coln[slot * m->ff_buf + j];
    }
    float w = m->ana_tok ? m->pair_w[m->order[g]] : 1.0f;
    switch (m->ana_mode) {
    case ANA_MAG:
        return mag;
    case ANA_AGREE:
        return a;
    case ANA_BOTH:
        return mag * a;
    case ANA_MAGW:
        return mag * col;
    case ANA_BOTHW:
        return mag * col * sqrtf(a);
    case ANA_TOKMAG:
        return w * mag;
    case ANA_TOKW:
        return w * mag * col;
    case ANA_TOKBOTH:
        return w * mag * col * sqrtf(a);
    default:
        return noise(m->ana_layer, g, j);
    }
}

struct ranked {
    float score;
    uint32_t g, j;
};

static int by_score(const void *a, const void *b)
{
    const struct ranked *x = a, *y = b;
    if (x->score != y->score)
        return x->score < y->score ? -1 : 1;
    if (x->g != y->g)
        return x->g < y->g ? -1 : 1;
    return x->j < y->j ? -1 : x->j > y->j ? 1 : 0;
}

/* Pair row g in the picture: red its agreements against the pair's highest,
   green its magnitudes (before any is off) against the largest; blue, its
   neurons switched off, set after. */
static void frame_pair(struct janas_llm_model *m, size_t g, const float *h,
                       uint32_t ff)
{
    if (!m->ana_frame || (g + 1) * m->ff_buf > m->ana_width)
        return;
    const float *agree = m->ana_agree ? m->ana_agree + g * m->ff_buf : NULL;
    float big = 0.0f, top = 0.0f;
    for (uint32_t j = 0; j < ff; j++) {
        if (fabsf(h[j]) > big)
            big = fabsf(h[j]);
        if (agree && agree[j] > top)
            top = agree[j];
    }
    uint8_t *px = m->ana_frame +
                  ((size_t)m->ana_layer * m->ana_width + g * m->ff_buf) * 3;
    for (uint32_t j = 0; j < ff; j++) {
        px[3 * j] = top > 0 ? (uint8_t)(255.0f * agree[j] / top + 0.5f) : 0;
        px[3 * j + 1] =
            big > 0 ? (uint8_t)(255.0f * sqrtf(fabsf(h[j]) / big) + 0.5f) : 0;
        px[3 * j + 2] = 0;
    }
}

static void frame_off(struct janas_llm_model *m, size_t g, uint32_t j)
{
    if (m->ana_frame && (g + 1) * m->ff_buf <= m->ana_width)
        m->ana_frame[((size_t)m->ana_layer * m->ana_width + g * m->ff_buf + j) *
                         3 +
                     2] = 160;
}

void janas_m_ana_mask(struct janas_llm_model *m, size_t g, float *h,
                      uint32_t ff)
{
    uint32_t k = (uint32_t)(m->ana_frac * (float)ff + 0.5f);
    struct ranked *r = NULL;
    if (m->ana_mode && k > 0 && (r = malloc(ff * sizeof(*r)))) {
        for (uint32_t j = 0; j < ff; j++)
            r[j] = (struct ranked){score(m, g, j, h[j]), (uint32_t)g, j};
        qsort(r, ff, sizeof(*r), by_score);
    }
    frame_pair(m, g, h, ff);
    for (uint32_t i = 0; r && i < k; i++) {
        frame_off(m, g, r[i].j);
        h[r[i].j] = 0.0f;
    }
    free(r);
}

static int by_token(const void *a, const void *b)
{
    const uint64_t *x = a, *y = b;
    return *x < *y ? -1 : *x > *y ? 1 : 0;
}

/* The ana_tok modes: the pair rows glist[0 .. n) of this pass grouped by
   token, each token's neurons ranked together - on h, or on the two-bit h
   with JANAS_ANA_COARSE - the fraction of them with the least score
   switched off (and, with JANAS_ANA_GRADE, the others given their levels),
   then every pair's h quantized. Pairs of a token computed in another pass
   (an expert read from the disk while the others ran) are ranked apart. */
static size_t rank_token(struct janas_llm_model *m, const uint64_t *key,
                         uint32_t i, uint32_t e, size_t total, int coarse,
                         struct ranked *r)
{
    uint32_t stride = m->ff_buf;
    size_t at = 0;
    for (uint32_t q = i; q < e; q++) {
        size_t g = (uint32_t)key[q];
        const float *h =
            coarse ? m->ana_coarse + g * stride : m->h + g * stride;
        for (uint32_t j = 0; j < m->hlen[g]; j++)
            r[at++] = (struct ranked){score(m, g, j, h[j]), (uint32_t)g, j};
    }
    qsort(r, total, sizeof(*r), by_score);
    return at;
}

void janas_m_ana_tok(struct janas_llm_model *m, uint32_t n)
{
    uint32_t stride = m->ff_buf;
    uint64_t *key = malloc(n * sizeof(uint64_t));
    if (key) {
        for (uint32_t i = 0; i < n; i++) {
            size_t g = m->glist[i];
            key[i] = ((uint64_t)m->pair_tok[m->order[g]] << 32) | (uint32_t)g;
        }
        qsort(key, n, sizeof(*key), by_token);
        for (uint32_t i = 0; i < n;) {
            uint32_t tok = (uint32_t)(key[i] >> 32), e = i;
            size_t total = 0;
            while (e < n && (uint32_t)(key[e] >> 32) == tok)
                total += m->hlen[(uint32_t)key[e++]];
            if (m->ana_xstats) {
                uint32_t head[3] = {(uint32_t)(m->ana_base + tok), m->ana_layer,
                                    m->d_model};
                fwrite(head, sizeof(head), 1, m->ana_xstats);
                fwrite(m->xn + (size_t)tok * m->d_model, sizeof(float),
                       m->d_model, m->ana_xstats);
            }
            struct ranked *r = malloc(total * sizeof(*r));
            if (!r) {
                i = e;
                continue;
            }
            for (uint32_t q = i; q < e; q++) {
                size_t g = (uint32_t)key[q];
                frame_pair(m, g, m->h + g * stride, m->hlen[g]);
            }
            size_t k = (size_t)(m->ana_frac * (float)total + 0.5f);
            rank_token(m, key, i, e, total, m->ana_coarse != NULL, r);
            /* with the two-bit choice, how much of the true one it keeps:
               levels mark the true choice's kept neurons for a moment */
            if (m->ana_coarse && m->ana_level == NULL) {
                struct ranked *t = malloc(total * sizeof(*t));
                uint8_t *mark = calloc((size_t)(e - i) * stride, 1);
                if (t && mark) {
                    rank_token(m, key, i, e, total, 0, t);
                    for (size_t q = k; q < total; q++)
                        for (uint32_t z = i; z < e; z++)
                            if ((uint32_t)key[z] == t[q].g) {
                                mark[(size_t)(z - i) * stride + t[q].j] = 1;
                                break;
                            }
                    for (size_t q = k; q < total; q++)
                        for (uint32_t z = i; z < e; z++)
                            if ((uint32_t)key[z] == r[q].g) {
                                m->ana_both_kept +=
                                    mark[(size_t)(z - i) * stride + r[q].j];
                                break;
                            }
                    m->ana_true_kept += total - k;
                }
                free(t);
                free(mark);
            }
            /* the levels: from the top, f6, f4 and f2 of the token's
               neurons; the rest, level 0, is what goes off below */
            if (m->ana_level) {
                size_t n6 = (size_t)(m->ana_grade[0] * (float)total + 0.5f),
                       n4 = (size_t)(m->ana_grade[1] * (float)total + 0.5f),
                       n2 = (size_t)(m->ana_grade[2] * (float)total + 0.5f);
                for (size_t q = 0; q < total; q++) {
                    size_t top = total - 1 - q;
                    m->ana_level[(size_t)r[q].g * stride + r[q].j] =
                        top < n6             ? 3
                        : top < n6 + n4      ? 2
                        : top < n6 + n4 + n2 ? 1
                                             : 0;
                }
            }
            /* the record first, with h as it was */
            uint8_t *keep = m->ana_stats ? malloc(stride) : NULL;
            for (uint32_t q = i; keep && q < e; q++) {
                size_t g = (uint32_t)key[q];
                uint32_t ff = m->hlen[g];
                memset(keep, 1, ff);
                for (size_t z = 0; z < k; z++)
                    if (r[z].g == g)
                        keep[r[z].j] = 0;
                float w = m->pair_w[m->order[g]];
                uint32_t head[5] = {(uint32_t)(m->ana_base + tok), m->ana_layer,
                                    m->ana_exp[g], ff, 0};
                memcpy(&head[4], &w, sizeof(w));
                fwrite(head, sizeof(head), 1, m->ana_stats);
                fwrite(m->h + g * stride, sizeof(float), ff, m->ana_stats);
                fwrite(keep, 1, ff, m->ana_stats);
            }
            free(keep);
            for (size_t q = 0; q < k; q++) {
                frame_off(m, r[q].g, r[q].j);
                m->h[(size_t)r[q].g * stride + r[q].j] = 0.0f;
            }
            free(r);
            i = e;
        }
        free(key);
    }
    for (uint32_t i = 0; i < n; i++) {
        size_t g = m->glist[i];
        janas_q8k_quantize_det(m->h + g * stride, m->hq + g * stride / JANAS_QK,
                               m->hlen[g]);
    }
}

/* The pair rows glist[0 .. n) of this pass whose gate and up go to two
   bits: in each token, its ana_gulow routed experts of least weight. */
void janas_m_ana_lowmark(struct janas_llm_model *m, uint32_t n)
{
    uint64_t *key = malloc(n * sizeof(uint64_t));
    if (!key)
        return;
    uint32_t nk = 0;
    for (uint32_t i = 0; i < n; i++) {
        size_t g = m->glist[i];
        m->ana_low[g] = 0;
        if (m->ana_exp[g] == m->n_expert) /* the shared expert: whole */
            continue;
        /* by token, then by weight (a positive float's bits sort as it) */
        float w = m->pair_w[m->order[g]];
        uint32_t wb;
        memcpy(&wb, &w, sizeof(wb));
        key[nk++] = ((uint64_t)m->pair_tok[m->order[g]] << 40) |
                    ((uint64_t)(wb >> 8) << 16) | (g & 0xFFFF);
    }
    qsort(key, nk, sizeof(*key), by_token);
    for (uint32_t i = 0; i < nk;) {
        uint32_t e = i;
        while (e < nk && (key[e] >> 40) == (key[i] >> 40))
            e++;
        for (uint32_t q = i; q < e && q < i + (uint32_t)m->ana_gulow; q++)
            m->ana_low[key[q] & 0xFFFF] = 1;
        i = e;
    }
    free(key);
}

/* Down at fewer bits for the neurons of the run's pairs whose level is 1 or
   2: their contribution corrected, after the product at full precision,
   by h times the difference of the row read at two or four bits. Q6_K_P:
   its planes; Q8_0: its bytes cut to two or four high bits; other types
   left whole. Rows split among the threads. */
struct grade_job {
    struct janas_llm_model *m;
    const uint8_t *down, *p1, *p0;
    const struct janas_jns_matrix *mx;
    uint32_t a, b;
};

static void cut_q8_0(uint8_t *blocks, size_t nb32, int bits)
{
    struct janas_block_q8_0 *k = (struct janas_block_q8_0 *)blocks;
    uint8_t keep = bits == 4 ? 0xF0 : 0xC0, mid = bits == 4 ? 0x08 : 0x20;
    for (size_t b = 0; b < nb32; b++)
        for (int i = 0; i < 32; i++)
            k[b].qs[i] = (int8_t)(((uint8_t)k[b].qs[i] & keep) | mid);
}

static void grade_worker(void *arg, int tid, int n_threads)
{
    struct grade_job *J = arg;
    struct janas_llm_model *m = J->m;
    uint32_t rows = J->mx->rows, cols = J->mx->cols, nb = cols / JANAS_QK;
    int planes = J->mx->type == JANAS_Q6_K_P, q8 = J->mx->type == JANAS_Q8_0;
    size_t bs = planes ? sizeof(struct janas_block_q6kp)
                       : janas_qtype_block_size((int)J->mx->type);
    float *w = malloc(3 * (size_t)cols * sizeof(float));
    uint8_t *tmp = malloc(nb * bs);
    if (!w || !tmp || !(planes || q8)) {
        free(w);
        free(tmp);
        return;
    }
    float *w6 = w, *w4 = w + cols, *w2 = w + 2 * (size_t)cols;
    for (uint32_t r = (uint32_t)tid; r < rows; r += (uint32_t)n_threads) {
        const uint8_t *row = J->down + (size_t)r * nb * bs;
        if (planes) {
            const struct janas_block_q6kp *base =
                (const struct janas_block_q6kp *)row;
            const uint8_t *p1 =
                J->p1 ? J->p1 + (size_t)r * nb * JANAS_Q6KP_PLANE : NULL;
            const uint8_t *p0 = J->p1 && J->p0
                                    ? J->p0 + (size_t)r * nb * JANAS_Q6KP_PLANE
                                    : NULL;
            janas_q6kp_dequantize(base, p1, p0, w6, cols);
            janas_q6kp_dequantize(base, p1, NULL, w4, cols);
            janas_q6kp_dequantize(base, NULL, NULL, w2, cols);
        } else {
            janas_dequantize(JANAS_Q8_0, row, w6, cols);
            memcpy(tmp, row, nb * bs);
            cut_q8_0(tmp, (size_t)nb * 8, 4);
            janas_dequantize(JANAS_Q8_0, tmp, w4, cols);
            memcpy(tmp, row, nb * bs);
            cut_q8_0(tmp, (size_t)nb * 8, 2);
            janas_dequantize(JANAS_Q8_0, tmp, w2, cols);
        }
        for (uint32_t g = J->a; g < J->b; g++) {
            const uint8_t *lv = m->ana_level + (size_t)g * m->ff_buf;
            const float *h = m->h + (size_t)g * m->ff_buf;
            double d = 0;
            for (uint32_t j = 0; j < cols; j++)
                if (lv[j] == 2)
                    d += (double)h[j] * (w4[j] - w6[j]);
                else if (lv[j] == 1)
                    d += (double)h[j] * (w2[j] - w6[j]);
            m->dout[(size_t)g * m->d_model + r] += (float)d;
        }
    }
    free(w);
    free(tmp);
}

void janas_m_ana_grade(struct janas_llm_model *m, const uint8_t *down,
                       const uint8_t *p1, const uint8_t *p0,
                       const struct janas_jns_matrix *mx, uint32_t a,
                       uint32_t b)
{
    if (!m->ana_level || mx->rows != m->d_model || mx->cols > m->ff_buf)
        return;
    struct grade_job J = {
        .m = m, .down = down, .p1 = p1, .p0 = p0, .mx = mx, .a = a, .b = b};
    janas_pool_run(m->compute, grade_worker, &J);
}

void janas_m_ana_frame(struct janas_llm_model *m, uint32_t n)
{
    m->ana_base += n;
    if (!m->ana_frame)
        return;
    if (n == 1 && m->ana_frames < m->ana_frames_max) {
        char path[600];
        snprintf(path, sizeof(path), "%s/frame_%05d.ppm", m->ana_dir,
                 m->ana_frames++);
        FILE *f = fopen(path, "wb");
        if (f) {
            fprintf(f, "P6\n%u %u\n255\n", m->ana_width, m->n_layer);
            fwrite(m->ana_frame, 3, (size_t)m->n_layer * m->ana_width, f);
            fclose(f);
        }
    }
    memset(m->ana_frame, 0, (size_t)m->n_layer * m->ana_width * 3);
}