Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

3.6 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
/*
 * vision_impl.h - the inside of the vision encoders: the encoder as loaded
 * (vision.c: Qwen3.5 and Qwen3-VL; vision_gemma.c: Gemma 4) and the
 * kernels they share (vision_kernels.c).
 */
#ifndef JANAS_LLM_VISION_IMPL_H
#define JANAS_LLM_VISION_IMPL_H

#include <stddef.h>
#include <stdint.h>

#include "jns.h"
#include "vision.h"
#include "vision_kernels.h"

#define MAX_BLOCKS 64

enum { VISION_QWEN = 1, VISION_GEMMA = 2 };

/* a block of Qwen's ViT */
struct vlayer {
    const float *ln1_w, *ln1_b, *ln2_w, *ln2_b;
    const uint16_t *qkv, *out, *up, *down;
    const float *qkv_b, *out_b, *up_b, *down_b;
    /* the same matrices transposed, for the products (owned) */
    uint16_t *t_qkv, *t_out, *t_up, *t_down;
};

/* a block of Gemma 4's: its norms, its linears transposed (owned) and the
   bounds each clamps its input and output to (q k v o gate up down, each
   in_min in_max out_min out_max) */
enum { GL_Q, GL_K, GL_V, GL_O, GL_GATE, GL_UP, GL_DOWN, GL_N };
struct glayer {
    const float *n_in, *n_post_att, *n_pre_ff, *n_post_ff, *q_norm, *k_norm;
    const float *clip; /* GL_N x 4 */
    uint16_t *t[GL_N];
};

struct janas_vision {
    struct janas_jns j;
    uint8_t *data; /* the resident tensors */
    int kind;      /* VISION_QWEN, VISION_GEMMA */
    int both_ways; /* the pictures' tokens attend both ways */
    uint32_t n_block, dim, ff, heads, patch, tpatch, merge, npos, proj;
    uint32_t min_pixels, max_pixels;
    float eps, theta, mean[3], stdv[3];
    int32_t tok_image, tok_start, tok_end;
    /* Qwen */
    const uint16_t *patch_w, *pos, *m_fc1, *m_fc2;
    const float *patch_b, *m_ln_w, *m_ln_b, *m_fc1_b, *m_fc2_b;
    struct vlayer l[MAX_BLOCKS];
    uint16_t *t_patch, *t_fc1, *t_fc2; /* transposed, owned */
    /* Gemma 4: the pooling, the positions' table (2 x pos_size rows: x,
       then y), the soft tokens a picture at most, the standardization
       (NULL: none), the projection to the language model (transposed) */
    uint32_t pool_k, pos_size, max_soft;
    const uint16_t *g_pos;
    const float *std_bias, *std_scale;
    struct glayer gl[MAX_BLOCKS];
    uint16_t *t_g_patch, *t_g_proj;
    /* Gemma 4 12B's, without blocks (vision_gemma_u.c): a 48x48 patch
       layer-normed, through a dense layer, normed, plus the positions,
       normed (u_ln: weight and bias of each norm, u_eps theirs) */
    int unified;
    float u_eps;
    const float *u_ln[3][2], *u_dense_b;
    uint16_t *t_u_dense;
};

/* a resident tensor of that name, type (0 float, 30 bfloat16) and shape,
   or NULL (vision.c) */
const void *janas_vision_tensor(struct janas_vision *v, const char *name,
                                uint32_t type, uint64_t rows, uint64_t cols);

/* vision_gemma.c: the rest of the open (NULL, or why not), the size of a
   picture, its encoding, what it owns freed */
const char *janas_vg_open(struct janas_vision *v);
void janas_vg_size(const struct janas_vision *v, int w, int h, int max_tokens,
                   int *rw, int *rh, int *nx, int *ny);
int janas_vg_encode(struct janas_vision *v, const struct janas_image *im,
                    int max_tokens, struct janas_pool *pool, float **out,
                    int *nx, int *ny, char *err, size_t err_len);
void janas_vg_close(struct janas_vision *v);
/* vision_gemma_u.c: the same for Gemma 4 12B's encoder without blocks */
const char *janas_vgu_open(struct janas_vision *v);
int janas_vgu_encode(struct janas_vision *v, const struct janas_image *im,
                     int max_tokens, struct janas_pool *pool, float **out,
                     int *nx, int *ny, char *err, size_t err_len);

#endif