Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

12.0 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
/*
 * llm_gen.c - greedy generation with and without speculative decoding.
 *
 * Generates the same continuation twice, first without speculation and then
 * with lookup drafts, checks that the tokens are identical, and reports the
 * speed of both and the acceptance of the drafts.
 *
 * Usage: llm_gen <model.jns> <prompt.int32> <n_prompt> <n_new> <k> [auto]
 *                [mtp=<mtp.jns>] [conf=<min probability of an MTP draft>]
 * With mtp= the drafts come from the model's MTP block instead of lookup.
 * With cpus=<list> (e.g. 0-11 or 18,19) the compute threads run on those
 * CPUs, one each (to run it beside another process on other cores).
 * LLM_GEN_DUMP=<path> writes the first run's tokens there (int32).
 */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "common/sysinfo.h"
#include "llm/generate.h"
#include "llm/model.h"

/* What the planner has learned: per tenth of guessed survival chance, how
   many drafts it tried and how many survived. */
static void report_calibration(const struct janas_gen_stats *s)
{
    int any = 0;
    for (int b = 0; b < 10; b++)
        any |= s->cal_tried[b] != 0;
    if (!any)
        return;
    printf("calibration, guessed chance -> tried, survived:");
    for (int b = 0; b < 10; b++)
        if (s->cal_tried[b])
            printf(" %.1f-%.1f: %u/%u (%.0f%%)", b / 10.0, (b + 1) / 10.0,
                   s->cal_won[b], s->cal_tried[b],
                   100.0 * s->cal_won[b] / s->cal_tried[b]);
    printf("\n");
}

static void report(const char *name, const struct janas_gen_stats *s)
{
    printf("%-8s prefill %u tokens %.2f s (%.1f token/s); decode %u tokens "
           "%.2f s (%.1f token/s), %u passes",
           name, s->prompt_tokens, s->prefill_seconds,
           s->prompt_tokens / s->prefill_seconds, s->new_tokens,
           s->decode_seconds, s->new_tokens / s->decode_seconds, s->passes);
    if (s->new_tokens)
        printf(", %u of %u forced (%.0f%%)", s->forced_tokens, s->new_tokens,
               100.0 * s->forced_tokens / s->new_tokens);
    if (s->sample_seconds > 0)
        printf(", sampling %.2f s (%.0f%%)", s->sample_seconds,
               100 * s->sample_seconds / s->decode_seconds);
    if (s->draft_seconds > 0)
        printf(", drafting %.2f s (%.0f%% of decode)", s->draft_seconds,
               100 * s->draft_seconds / s->decode_seconds);
    if (s->spec_passes)
        printf(", %u verifying: %u of %u drafted accepted (%.0f%%), %.2f "
               "tokens per pass",
               s->spec_passes, s->accepted, s->drafted,
               100.0 * s->accepted / s->drafted,
               (double)s->new_tokens / s->passes);
    if (s->auto_off_passes)
        printf(", %u passes with speculation paused", s->auto_off_passes);
    printf("\n");
}

/* By position in the chain: how often the draft there was accepted, of all
   verified there and of those whose predecessor was accepted; and by the
   tokens in a pass, its mean time against a pass of one in the run without
   drafts (plain). */
static void report_chain(const struct janas_gen_stats *s,
                         const struct janas_gen_stats *plain)
{
    if (!s->pos_tried[0])
        return;
    printf("chain, position: accepted/verified (%%, %% given the one "
           "before):");
    for (int i = 0; i < JANAS_GEN_MAX_K && s->pos_tried[i]; i++) {
        uint32_t before = i ? s->pos_won[i - 1] : s->pos_tried[0];
        printf(" %d: %u/%u (%.0f%%, %.0f%%)", i + 1, s->pos_won[i],
               s->pos_tried[i], 100.0 * s->pos_won[i] / s->pos_tried[i],
               before ? 100.0 * s->pos_won[i] / before : 0.0);
    }
    printf("\n");
    double one = plain->pass_count[1]
                     ? plain->pass_seconds[1] / plain->pass_count[1]
                     : 0;
    printf("pass, tokens: passes, ms (x a pass of one):");
    for (int n = 1; n < JANAS_GEN_MAX_K + 2; n++)
        if (s->pass_count[n]) {
            double t = s->pass_seconds[n] / s->pass_count[n];
            printf(" %d: %u, %.1f (%.2f)", n, s->pass_count[n], t * 1e3,
                   one > 0 ? t / one : 0.0);
        }
    printf("\n");
}

int main(int argc, char **argv)
{
    if (argc < 6) {
        fprintf(stderr,
                "usage: llm_gen <model.jns> <prompt.int32> <n_prompt> "
                "<n_new> <k> [auto] [mtp=<mtp.jns>] [conf=<p>] "
                "[threads=<n>] [cache=<MiB>] [ctx=<n>] [temp=<t>] [seed=<n>] "
                "[draft=<small.jns>]\n");
        return 2;
    }
    int32_t prompt[8192];
    FILE *f = fopen(argv[2], "rb");
    if (!f)
        return 1;
    size_t avail = fread(prompt, sizeof(int32_t), 8192, f);
    fclose(f);
    uint32_t np = (uint32_t)atoi(argv[3]), nn = (uint32_t)atoi(argv[4]);
    if (np == 0 || np > avail || nn == 0)
        return 2;

    int use_auto = 0;
    const char *mtp_path = NULL;
    float min_conf = 0.0f;
    /* the product picks its own thread count; the default here is the one
       this test has always used, so old numbers stay comparable */
    int n_compute = 12, n_ctx = 4096;
    uint64_t cache_mib = 20480;
    float temp = 0.0f; /* greedy, so that the two runs can be compared */
    const char *draft_path = NULL;
    uint64_t seed = 20260922;
    float top_p = 0.8f;
    int cpu_list[64], n_list = 0;
    for (int i = 6; i < argc; i++) {
        if (strcmp(argv[i], "auto") == 0)
            use_auto = 1;
        else if (strncmp(argv[i], "mtp=", 4) == 0)
            mtp_path = argv[i] + 4;
        else if (strncmp(argv[i], "conf=", 5) == 0)
            min_conf = (float)atof(argv[i] + 5);
        else if (strncmp(argv[i], "threads=", 8) == 0)
            n_compute = atoi(argv[i] + 8);
        else if (strncmp(argv[i], "cache=", 6) == 0)
            cache_mib = (uint64_t)atol(argv[i] + 6);
        else if (strncmp(argv[i], "ctx=", 4) == 0)
            n_ctx = atoi(argv[i] + 4);
        else if (strncmp(argv[i], "temp=", 5) == 0)
            temp = (float)atof(argv[i] + 5);
        else if (strncmp(argv[i], "draft=", 6) == 0)
            draft_path = argv[i] + 6;
        else if (strncmp(argv[i], "seed=", 5) == 0)
            seed = strtoull(argv[i] + 5, NULL, 10);
        else if (strncmp(argv[i], "cpus=", 5) == 0) {
            /* a, a-b, comma separated */
            for (const char *c = argv[i] + 5; *c && n_list < 64;) {
                char *e;
                long a = strtol(c, &e, 10), b = a;
                if (e == c)
                    break;
                if (*e == '-')
                    b = strtol(e + 1, &e, 10);
                for (long k = a; k <= b && n_list < 64; k++)
                    cpu_list[n_list++] = (int)k;
                c = *e == ',' ? e + 1 : e;
            }
        } else if (strncmp(argv[i], "top_p=", 6) == 0)
            top_p = (float)atof(argv[i] + 6);
        else if (strncmp(argv[i], "seed=", 5) == 0)
            seed = (uint64_t)strtoull(argv[i] + 5, NULL, 10);
    }
    int icpus[8] = {12, 13, 14, 15, 16, 17, 18, 19};
    struct janas_cpu_layout lay;
    int ccpus[12] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11};
    const int *compute = ccpus;
    if (n_list > 0) {
        compute = cpu_list;
        n_compute = n_list;
    } else if (n_compute != 12 && janas_cpu_layout(NULL, 0, &lay) == 0) {
        if (n_compute < 1 || n_compute > lay.n)
            n_compute = lay.n;
        compute = lay.cpus;
    }
    struct janas_llm_options mo = {.cache_bytes = cache_mib << 20,
                                   .n_ctx = (uint32_t)n_ctx,
                                   .compute_cpus = compute,
                                   .n_compute = n_compute,
                                   .io_cpus = icpus,
                                   .n_io = 8,
                                   .mtp_path = mtp_path};
    char err[256];
    struct janas_llm_model *m =
        janas_llm_model_load(argv[1], &mo, err, sizeof(err));
    if (!m) {
        fprintf(stderr, "%s: %s\n", argv[1], err);
        return 1;
    }
    /* the drafting model shares the target's pool: two of them would take
       the cores from each other */
    struct janas_llm_model *draft = NULL;
    if (draft_path) {
        struct janas_llm_options dmo = mo;
        dmo.mtp_path = NULL;
        dmo.cache_bytes = (uint64_t)2048 << 20;
        dmo.shared_compute = janas_llm_model_compute(m);
        draft = janas_llm_model_load(draft_path, &dmo, err, sizeof(err));
        if (!draft) {
            fprintf(stderr, "%s: %s\n", draft_path, err);
            return 1;
        }
    }
    int32_t *a = malloc(nn * sizeof(int32_t)),
            *b = malloc(nn * sizeof(int32_t));
    /* the same seed on both runs: at a temperature above zero the two
       still draw the same tokens unless the drafts change the outcome */
    struct janas_gen_options off = {.spec_mode = JANAS_SPEC_OFF,
                                    .spec_k = 1,
                                    .eos = -1,
                                    .temperature = temp,
                                    /* the product's own sampling */
                                    .top_k = temp > 0 ? 20 : 0,
                                    .top_p = temp > 0 ? top_p : 0.0f,
                                    .seed = seed};
    int has_mtp = janas_llm_model_has_mtp(m); /* a file, or in the model */
    struct janas_gen_options on = {.spec_mode = draft     ? JANAS_SPEC_DRAFT
                                                : has_mtp ? JANAS_SPEC_MTP
                                                          : JANAS_SPEC_LOOKUP,
                                   .draft = draft,
                                   .spec_k = atoi(argv[5]),
                                   .spec_auto = use_auto,
                                   .eos = -1,
                                   .spec_min_conf = min_conf,
                                   .temperature = temp,
                                   .top_k = temp > 0 ? 20 : 0,
                                   .top_p = temp > 0 ? top_p : 0.0f,
                                   .seed = seed};
    struct janas_gen_stats sa, sb;
    /* warm-up run so both measured runs see the same expert cache */
    int na = janas_llm_generate(m, prompt, np, nn, &off, a, &sa);
    na = janas_llm_generate(m, prompt, np, nn, &off, a, &sa);
    int nb = janas_llm_generate(m, prompt, np, nn, &on, b, &sb);
    if (na < 0 || nb < 0) {
        fprintf(stderr, "generation failed\n");
        return 1;
    }
    /* LLM_GEN_DUMP=<path>: the first run's tokens there (int32), to read
       or compare against another run's */
    const char *dump = getenv("LLM_GEN_DUMP");
    if (dump) {
        FILE *f = fopen(dump, "wb");
        if (!f || fwrite(a, sizeof(int32_t), (size_t)na, f) != (size_t)na)
            fprintf(stderr, "%s: not written\n", dump);
        if (f)
            fclose(f);
    }
    printf("first tokens:");
    for (int i = 0; i < 12 && i < na; i++)
        printf(" %d", a[i]);
    printf("\n");
    printf("first tokens:");
    for (int i = 0; i < 12 && i < nb; i++)
        printf(" %d", b[i]);
    printf("\n");
    report("normal", &sa);
    report(draft     ? "draft"
           : has_mtp ? (on.spec_auto ? "mtp+a" : "mtp")
                     : (on.spec_auto ? "lookup+a" : "lookup"),
           &sb);
    report_calibration(&sb);
    report_chain(&sb, &sa);
    int same = na == nb && memcmp(a, b, (size_t)na * sizeof(int32_t)) == 0;
    /* where they part: on the CPU the drafts never change the text, on
       the GPU its sums in another order may (its quality is measured
       against a reference instead, llm_eval) */
    int first = 0;
    while (first < na && first < nb && a[first] == b[first])
        first++;
    printf("tokens identical: %s (%d and %d", same ? "yes" : "NO", na, nb);
    if (!same)
        printf(", the first difference at token %d", first);
    printf("); speed-up %.2fx\n", (sb.new_tokens / sb.decode_seconds) /
                                      (sa.new_tokens / sa.decode_seconds));
    free(a);
    free(b);
    char tune[512];
    janas_llm_model_tuning(m, tune, sizeof(tune));
    printf("tuning: %s\n", tune);
    janas_llm_model_free(m);
    return same ? 0 : 1;
}