Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
/* qwenimage_vae.h -- the Qwen-Image-2.1 VAE decoder (diffusers AutoencoderKLQwenImage21),
* one image at a time, pure C on top of qi_gemm.
*
* The diffusers class is Wan's causal video VAE specialised to images. For one
* frame its time machinery folds away, and this file only keeps what a single
* frame runs:
* - every CausalConv3d is a zero-padded Conv2d (the class squeezes the time
* axis and calls nn.Conv2d), so the weights are plain [Cout][Cin][k][k];
* - the time_conv of an "upsample3d" resample only runs from the second chunk
* on (the first one stores "Rep" in the feature cache), so it is never loaded;
* - DupUp3D with first_chunk=True keeps temporal slot factor_t-1 only, which
* makes the up-block shortcut a gather of input channels (qiv_dup_table).
* Path: z*std + mean, post_quant_conv (1x1), conv_in, mid block (resnet,
* single-head attention over all pixels, resnet), residual up blocks, RMS norm
* + SiLU, conv_out, clamp to [-1, 1], then VaeImageProcessor.postprocess
* ((x*0.5 + 0.5).clamp(0, 1) * 255, numpy round-half-even, uint8).
*
* Activations are channel-last [H][W][C]. A 3x3 conv is one qi_gemm per band of
* output rows over a 3x (not 9x) im2col, see qiv_conv3; the weight is permuted
* at load to [Cout][kx][ky][Cin] to match. The im2col only exists for a band of
* output rows (qiv_col_floats), and the input rows a band has consumed
* are handed back to the OS when nothing reads them later (qiv_release): at
* 1024x1024 the last up blocks hold 288-channel maps of 1.2 GB each, and without
* that the decode would not fit next to the rest of the pipeline.
*
* Cost: about 4.2 TFLOP at 512x512, growing with the pixel count; measured on
* ds (Zen 4, 8 threads) 81% of the time is qi_gemm at 700-800 GFLOP/s, the rest
* is the RMS norm + SiLU prologue, the im2col copy and the epilogues. Peak
* memory is the weights (1.0 GB f32) plus about 1.8 GB per megapixel.
*
* Interface: qiv_load(dir) / qiv_decode(v, z, h, w, rgba, out_f) / qiv_free(v);
* qiv_set_progress for a callback after each stage (8 for the real model).
*/
#ifndef COLIBRI_QWENIMAGE_VAE_H
#define COLIBRI_QWENIMAGE_VAE_H
#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <time.h>
#include "st.h"
#include "json.h"
#include "qi_gemm.h"
#ifdef __linux__
#include <sys/mman.h>
#include <unistd.h>
#endif
#ifdef _OPENMP
#include <omp.h>
#endif
#define QIV_MAX_BLOCKS 8
#define QIV_MAX_RES 8
/* im2col band budget in floats (64 MB): large enough that the GEMM sees
* thousands of rows, small next to the activations it serves. The attention
* score block gets 16 MB. Variables so a test can shrink both to one row and
* cover the band edges on a tiny model. */
static size_t qiv_col_floats = (size_t)16 << 20;
static size_t qiv_attn_floats = (size_t)4 << 20;
typedef struct {
int cin, cout, k; /* k = 1 or 3 */
float *w, *b; /* w: [cout][kx][ky][cin] */
QiMat m;
void *vk; int bo; /* qwenimage_vae_vk.h: the device's copy of w, b's place in its parameters */
} QivConv;
typedef struct {
int cin, cout;
float *g1, *g2; /* RMS norm gammas, [cin] and [cout] */
int g1o, g2o; /* their places in the device's parameters (qwenimage_vae_vk.h) */
QivConv c1, c2, sc; /* sc only when cin != cout */
} QivRes;
typedef struct {
int cin, cout, up, ft; /* up: 2x upsample + DupUp3D shortcut; ft: its factor_t */
int nres;
QivRes res[QIV_MAX_RES];
QivConv upc; /* the conv after the nearest 2x upsample */
int *dup; /* [cout][2][2] -> input channel of the DupUp3D shortcut */
} QivBlock;
typedef struct QiVae {
int z_dim, out_ch, scale; /* scale: pixels per latent along each side (16) */
float *mean, *std; /* [z_dim] from config.json */
QivConv pqc, conv_in;
QivRes mid[2];
float *attn_g;
QivConv qkv, proj;
int nblk;
QivBlock blk[QIV_MAX_BLOCKS];
float *g_out;
int g_outo; /* its place in the device's parameters */
QivConv conv_out;
size_t wbytes; /* resident weight bytes */
void (*progress)(void *ud, int step, int steps);
void *progress_ud;
} QiVae;
static double qiv_now(void){
struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts);
return ts.tv_sec + ts.tv_nsec * 1e-9;
}
/* ---------------------------------------------------------------- memory -- */
/* Hand the whole pages of buf[from, upto) (in floats) back to the OS: the caller
* never reads them again. Only for large buffers, which glibc serves with their
* own mmap; a page strictly inside a buffer belongs to it alone either way, and
* free() later only writes (never reads) near the chunk edges. Elsewhere this is
* a no-op and the peak is simply higher. */
static void qiv_release(float *buf, size_t total, size_t from, size_t upto){
#if defined(__linux__) && defined(MADV_DONTNEED)
if (!buf || total < (8u << 20) || upto <= from) return;
static long pg = 0;
if (!pg) { pg = sysconf(_SC_PAGESIZE); if (pg <= 0) pg = 4096; }
uintptr_t a = (uintptr_t)(buf + from), e = (uintptr_t)(buf + upto);
a = (a + pg - 1) & ~(uintptr_t)(pg - 1);
e = e & ~(uintptr_t)(pg - 1);
if (e > a) madvise((void *)a, e - a, MADV_DONTNEED);
#else
(void)buf; (void)total; (void)from; (void)upto;
#endif
}
static float *qiv_alloc(size_t n){
float *p = (float *)malloc(n * sizeof(float) + 64);
if (!p) fprintf(stderr, "qwenimage vae: out of memory (%.1f MB)\n", n * 4.0 / 1e6);
return p;
}
/* ----------------------------------------------------------------- loading -- */
static float *qiv_read(shards *S, const char *name, int64_t numel, char *err, size_t errn){
st_tensor *t = st_find(S, name);
if (!t) { snprintf(err, errn, "missing tensor %s", name); return NULL; }
if (t->numel != numel) {
snprintf(err, errn, "tensor %s has %lld elements, expected %lld", name,
(long long)t->numel, (long long)numel);
return NULL;
}
float *p = qiv_alloc((size_t)numel);
if (!p) { snprintf(err, errn, "out of memory reading %s", name); return NULL; }
st_read_f32(S, name, p, 1);
return p;
}
static int qiv_load_conv(shards *S, QiVae *v, const char *pfx, int cin, int cout, int k,
QivConv *c, char *err, size_t errn){
char nm[256];
c->cin = cin; c->cout = cout; c->k = k;
snprintf(nm, sizeof nm, "%s.weight", pfx);
float *w = qiv_read(S, nm, (int64_t)cout * cin * k * k, err, errn);
if (!w) return -1;
if (k > 1) {
/* [cout][cin][ky][kx] -> [cout][kx][ky][cin], the order qiv_conv3 reads taps in */
float *p = qiv_alloc((size_t)cout * cin * k * k);
if (!p) { free(w); snprintf(err, errn, "out of memory"); return -1; }
for (int o = 0; o < cout; o++)
for (int i = 0; i < cin; i++)
for (int ky = 0; ky < k; ky++)
for (int kx = 0; kx < k; kx++)
p[((size_t)o * k * k + kx * k + ky) * cin + i] = w[((size_t)o * cin + i) * k * k + ky * k + kx];
free(w); w = p;
}
snprintf(nm, sizeof nm, "%s.bias", pfx);
float *b = qiv_read(S, nm, cout, err, errn);
if (!b) { free(w); return -1; }
c->w = w; c->b = b;
c->m = (QiMat){ QI_F32, cout, cin * k * k, w, NULL };
v->wbytes += ((size_t)cout * cin * k * k + cout) * sizeof(float);
return 0;
}
static int qiv_load_res(shards *S, QiVae *v, const char *pfx, int cin, int cout, QivRes *r,
char *err, size_t errn){
char nm[256];
r->cin = cin; r->cout = cout;
snprintf(nm, sizeof nm, "%s.norm1.gamma", pfx);
if (!(r->g1 = qiv_read(S, nm, cin, err, errn))) return -1;
snprintf(nm, sizeof nm, "%s.norm2.gamma", pfx);
if (!(r->g2 = qiv_read(S, nm, cout, err, errn))) return -1;
v->wbytes += (size_t)(cin + cout) * sizeof(float);
snprintf(nm, sizeof nm, "%s.conv1", pfx);
if (qiv_load_conv(S, v, nm, cin, cout, 3, &r->c1, err, errn)) return -1;
snprintf(nm, sizeof nm, "%s.conv2", pfx);
if (qiv_load_conv(S, v, nm, cout, cout, 3, &r->c2, err, errn)) return -1;
if (cin != cout) {
snprintf(nm, sizeof nm, "%s.conv_shortcut", pfx);
if (qiv_load_conv(S, v, nm, cin, cout, 1, &r->sc, err, errn)) return -1;
}
return 0;
}
static void qiv_free_conv(QivConv *c){ free(c->w); free(c->b); c->w = c->b = NULL; }
static void qiv_free_res(QivRes *r){
free(r->g1); free(r->g2); r->g1 = r->g2 = NULL;
qiv_free_conv(&r->c1); qiv_free_conv(&r->c2); qiv_free_conv(&r->sc);
}
static void qiv_free(QiVae *v){
if (!v) return;
free(v->mean); free(v->std);
qiv_free_conv(&v->pqc); qiv_free_conv(&v->conv_in);
qiv_free_res(&v->mid[0]); qiv_free_res(&v->mid[1]);
free(v->attn_g); qiv_free_conv(&v->qkv); qiv_free_conv(&v->proj);
for (int i = 0; i < v->nblk; i++) {
for (int j = 0; j < v->blk[i].nres; j++) qiv_free_res(&v->blk[i].res[j]);
qiv_free_conv(&v->blk[i].upc);
free(v->blk[i].dup);
}
free(v->g_out); qiv_free_conv(&v->conv_out);
free(v);
}
static int qiv_json_int(jval *cfg, const char *key, int def){
jval *j = json_get(cfg, key);
return (j && j->t == J_NUM) ? (int)j->num : def;
}
/* DupUp3D for one frame with first_chunk: repeat_interleave(repeats) over
* channels, view [cout][ft][2][2], keep time slot ft-1. So output channel o at
* sub-pixel (sy, sx) reads input channel (o*4*ft + (ft-1)*4 + 2*sy + sx) / repeats. */
static int *qiv_dup_table(int cin, int cout, int ft){
int factor = 4 * ft;
if ((cout * factor) % cin) return NULL;
int repeats = cout * factor / cin;
int *t = (int *)malloc(sizeof(int) * cout * 4);
if (!t) return NULL;
for (int o = 0; o < cout; o++)
for (int s = 0; s < 4; s++)
t[o * 4 + s] = (o * factor + (ft - 1) * 4 + s) / repeats;
return t;
}
static QiVae *qiv_load(const char *vae_dir){
char err[512] = "", path[4096];
QiVae *v = (QiVae *)calloc(1, sizeof(QiVae));
shards S; int have_s = 0;
jval *cfg = NULL;
char *text = NULL;
if (!v) { fprintf(stderr, "qwenimage vae: out of memory\n"); return NULL; }
if (snprintf(path, sizeof path, "%s/config.json", vae_dir) >= (int)sizeof path) {
snprintf(err, sizeof err, "path too long"); goto fail; }
FILE *f = fopen(path, "rb");
if (!f) { snprintf(err, sizeof err, "cannot open %.400s", path); goto fail; }
fseek(f, 0, SEEK_END); long n = ftell(f); fseek(f, 0, SEEK_SET);
text = (char *)malloc(n > 0 ? n + 1 : 1);
if (!text || n <= 0 || fread(text, 1, n, f) != (size_t)n) {
fclose(f); snprintf(err, sizeof err, "cannot read %.400s", path); goto fail; }
fclose(f); text[n] = 0;
cfg = json_parse_checked(text);
if (!cfg || cfg->t != J_OBJ) { snprintf(err, sizeof err, "%.400s is not a JSON object", path); goto fail; }
v->z_dim = qiv_json_int(cfg, "z_dim", 0);
v->out_ch = qiv_json_int(cfg, "out_channels", 3);
int base = qiv_json_int(cfg, "decoder_base_dim", 0);
if (base <= 0) base = qiv_json_int(cfg, "base_dim", 0);
int nrb = qiv_json_int(cfg, "num_res_blocks", 2);
jval *jm = json_get(cfg, "dim_mult"), *jt = json_get(cfg, "temperal_downsample");
jval *jr = json_get(cfg, "is_residual"), *jp = json_get(cfg, "patch_size");
jval *jmean = json_get(cfg, "latents_mean"), *jstd = json_get(cfg, "latents_std");
if (v->z_dim <= 0 || base <= 0 || nrb < 0 || nrb + 1 > QIV_MAX_RES || !jm || jm->t != J_ARR
|| jm->len < 1 || jm->len > QIV_MAX_BLOCKS || !jt || jt->t != J_ARR || jt->len != jm->len - 1
|| v->out_ch <= 0) {
snprintf(err, sizeof err, "%.400s: missing or unexpected z_dim/base_dim/dim_mult/temperal_downsample", path);
goto fail;
}
/* The non-residual (Wan 2.1 style) decoder and patchified output are other
* checkpoints; refuse them rather than decode garbage. */
if (!jr || jr->t != J_BOOL || !jr->boolean) { snprintf(err, sizeof err, "only is_residual=true is supported"); goto fail; }
if (jp && jp->t != J_NULL) { snprintf(err, sizeof err, "patch_size must be null"); goto fail; }
if (!jmean || jmean->t != J_ARR || jmean->len != v->z_dim || !jstd || jstd->t != J_ARR || jstd->len != v->z_dim) {
snprintf(err, sizeof err, "latents_mean/latents_std must have z_dim=%d entries", v->z_dim); goto fail; }
v->mean = qiv_alloc(v->z_dim); v->std = qiv_alloc(v->z_dim);
if (!v->mean || !v->std) { snprintf(err, sizeof err, "out of memory"); goto fail; }
for (int i = 0; i < v->z_dim; i++) { v->mean[i] = (float)jmean->kids[i]->num; v->std[i] = (float)jstd->kids[i]->num; }
int nm = jm->len, dims[QIV_MAX_BLOCKS + 1], tup[QIV_MAX_BLOCKS];
/* Every width is a product of two config numbers that size buffers and a
* divisor in qiv_dup_table: positive and bounded, or refused. */
for (int i = 0; i < nm; i++)
if (jm->kids[i]->t != J_NUM || jm->kids[i]->num < 1 || jm->kids[i]->num > 64 || base > 4096) {
snprintf(err, sizeof err, "dim_mult entries must be whole numbers from 1 to 64 (base %d)", base); goto fail; }
for (int i = 0; i < nm - 1; i++)
if (jt->kids[i]->t != J_BOOL) { snprintf(err, sizeof err, "temperal_downsample must hold booleans"); goto fail; }
if (v->z_dim > 4096) { snprintf(err, sizeof err, "z_dim %d is out of range", v->z_dim); goto fail; }
dims[0] = base * (int)jm->kids[nm - 1]->num;
for (int i = 0; i < nm; i++) dims[i + 1] = base * (int)jm->kids[nm - 1 - i]->num;
for (int i = 0; i < nm - 1; i++) tup[i] = jt->kids[nm - 2 - i]->boolean; /* temperal_upsample = reversed */
{
DIR *d = opendir(vae_dir);
if (!d) { snprintf(err, sizeof err, "cannot open directory %.400s", vae_dir); goto fail; }
closedir(d);
}
st_init(&S, vae_dir); have_s = 1;
if (S.n == 0) { snprintf(err, sizeof err, "no safetensors in %.400s", vae_dir); goto fail; }
if (qiv_load_conv(&S, v, "post_quant_conv", v->z_dim, v->z_dim, 1, &v->pqc, err, sizeof err)) goto fail;
if (qiv_load_conv(&S, v, "decoder.conv_in", v->z_dim, dims[0], 3, &v->conv_in, err, sizeof err)) goto fail;
for (int i = 0; i < 2; i++) {
char p[128]; snprintf(p, sizeof p, "decoder.mid_block.resnets.%d", i);
if (qiv_load_res(&S, v, p, dims[0], dims[0], &v->mid[i], err, sizeof err)) goto fail;
}
if (!(v->attn_g = qiv_read(&S, "decoder.mid_block.attentions.0.norm.gamma", dims[0], err, sizeof err))) goto fail;
if (qiv_load_conv(&S, v, "decoder.mid_block.attentions.0.to_qkv", dims[0], 3 * dims[0], 1, &v->qkv, err, sizeof err)) goto fail;
if (qiv_load_conv(&S, v, "decoder.mid_block.attentions.0.proj", dims[0], dims[0], 1, &v->proj, err, sizeof err)) goto fail;
v->scale = 1;
v->nblk = nm;
for (int i = 0; i < nm; i++) {
QivBlock *b = &v->blk[i];
b->cin = dims[i]; b->cout = dims[i + 1];
b->up = i != nm - 1;
b->ft = b->up && tup[i] ? 2 : 1;
b->nres = nrb + 1;
for (int j = 0; j < b->nres; j++) {
char p[128]; snprintf(p, sizeof p, "decoder.up_blocks.%d.resnets.%d", i, j);
if (qiv_load_res(&S, v, p, j ? b->cout : b->cin, b->cout, &b->res[j], err, sizeof err)) goto fail;
}
if (b->up) {
char p[128]; snprintf(p, sizeof p, "decoder.up_blocks.%d.upsampler.resample.1", i);
if (qiv_load_conv(&S, v, p, b->cout, b->cout, 3, &b->upc, err, sizeof err)) goto fail;
if (!(b->dup = qiv_dup_table(b->cin, b->cout, b->ft))) {
snprintf(err, sizeof err, "up block %d: DupUp3D %d->%d channels is not a whole repeat", i, b->cin, b->cout);
goto fail;
}
v->scale *= 2;
}
}
if (!(v->g_out = qiv_read(&S, "decoder.norm_out.gamma", dims[nm], err, sizeof err))) goto fail;
if (qiv_load_conv(&S, v, "decoder.conv_out", dims[nm], v->out_ch, 3, &v->conv_out, err, sizeof err)) goto fail;
v->wbytes += (size_t)(dims[0] + dims[nm]) * sizeof(float);
if (v->out_ch != 4 && v->out_ch != 3) { snprintf(err, sizeof err, "out_channels=%d, expected 3 or 4", v->out_ch); goto fail; }
st_destroy(&S);
json_free(cfg); free(text);
return v;
fail:
fprintf(stderr, "qwenimage vae: %s\n", err[0] ? err : "load failed");
if (have_s) st_destroy(&S);
json_free(cfg); free(text);
qiv_free(v);
return NULL;
}
static void qiv_set_progress(QiVae *v, void (*cb)(void *ud, int step, int steps), void *ud){
v->progress = cb; v->progress_ud = ud;
}
/* --------------------------------------------------------------- compute -- */
/* RMS_norm: F.normalize over channels (x / max(||x||, 1e-12)), times sqrt(C),
* times gamma; then SiLU when asked. Same operation order as the module. */
static inline void qiv_norm_px(float *d, const float *s, int C, const float *g, float scale, int silu){
double ss = 0;
for (int c = 0; c < C; c++) ss += (double)s[c] * s[c];
float den = (float)sqrt(ss);
if (den < 1e-12f) den = 1e-12f;
for (int c = 0; c < C; c++) {
float v = s[c] / den * scale * g[c];
d[c] = silu ? v / (1.f + expf(-v)) : v;
}
}
/* What a conv adds to its output before storing it, and which inputs it may
* give back as it goes (the caller's promise that nothing reads them later). */
typedef struct {
const float *res; /* y += res, same layout as y; may be y itself */
const float *scx; /* y += sc(scx): a 1x1 conv of the block input at output resolution */
const QivConv *sc;
const float *dup; /* y += DupUp3D(dup): dup is at half the output resolution */
int dupC; const int *dupt;
int rel_x, rel_scx, rel_dup;
} QivEpi;
/* y[Ho][Wo][cout] = conv3x3(pre(x)) (+ epilogue), with pre = RMS norm + SiLU if
* gamma, and x nearest-upsampled 2x first if up. Processed in bands of output
* rows. P holds the band's input rows after pre, with a one-row halo and a zero
* column on each side (width Wp = Wo + 2). Q interleaves three consecutive P
* rows: Q[r][u][ky] = P[r + ky][u]. Then the nine taps of output pixel (r, u)
* are the 9*C contiguous floats at Q[r][u], ordered [kx][ky][c], and the next
* pixel starts 3*C floats later: a GEMM with ldx = 3*C over the rows r*Wp + u
* reads them in place. Q costs 3 copies of the input instead of the 9 of a full
* im2col; the two rows per band that straddle a line end are computed and
* dropped. */
static int qiv_conv3(const QivConv *cv, float *x, int H, int W, const float *gamma, int up,
float *y, const QivEpi *ep){
const int C = cv->cin, Co = cv->cout, C3 = 3 * C;
const int Ho = H << up, Wo = W << up, Wp = Wo + 2;
const float scale = (float)sqrt((double)C);
const size_t rowq = (size_t)Wp * C3;
int band = (int)(qiv_col_floats / rowq);
if (band < 1) band = 1;
if (band > Ho) band = Ho;
float *P = qiv_alloc((size_t)(band + 2) * Wp * C);
float *Q = qiv_alloc((size_t)band * rowq);
float *T = qiv_alloc((size_t)band * Wp * Co);
float *T2 = ep && ep->sc ? qiv_alloc((size_t)band * Wo * Co) : NULL;
if (!P || !Q || !T || (ep && ep->sc && !T2)) { free(P); free(Q); free(T); free(T2); return -1; }
const size_t xtot = (size_t)H * W * C;
const int scC = ep && ep->sc ? ep->sc->cin : 0;
size_t xrel = 0, screl = 0, duprel = 0;
for (int y0 = 0; y0 < Ho; y0 += band) {
const int nb = Ho - y0 < band ? Ho - y0 : band;
/* P row r = virtual input row y0-1+r */
#pragma omp parallel for collapse(2) schedule(static)
for (int r = 0; r < nb + 2; r++)
for (int u = 0; u < Wp; u++) {
float *d = P + ((size_t)r * Wp + u) * C;
int vr = y0 - 1 + r, vu = u - 1;
if (vr < 0 || vr >= Ho || vu < 0 || vu >= Wo) { memset(d, 0, sizeof(float) * C); continue; }
const float *s = x + ((size_t)(vr >> up) * W + (vu >> up)) * C;
if (gamma) qiv_norm_px(d, s, C, gamma, scale, 1);
else memcpy(d, s, sizeof(float) * C);
}
#pragma omp parallel for collapse(2) schedule(static)
for (int r = 0; r < nb; r++)
for (int u = 0; u < Wp; u++) {
float *d = Q + ((size_t)r * Wp + u) * C3;
for (int ky = 0; ky < 3; ky++)
memcpy(d + ky * C, P + ((size_t)(r + ky) * Wp + u) * C, sizeof(float) * C);
}
qi_gemm_ld(T, Co, Q, C3, nb * Wp - 2, &cv->m, cv->b);
if (T2) qi_gemm(T2, ep->scx + (size_t)y0 * Wo * scC, nb * Wo, &ep->sc->m, ep->sc->b);
#pragma omp parallel for collapse(2) schedule(static)
for (int r = 0; r < nb; r++)
for (int u = 0; u < Wo; u++) {
const int p = r * Wo + u;
float *t = T + ((size_t)r * Wp + u) * Co;
size_t gp = (size_t)y0 * Wo + p;
if (T2) { const float *t2 = T2 + (size_t)p * Co; for (int o = 0; o < Co; o++) t[o] += t2[o]; }
if (ep && ep->dup) {
int Y = y0 + r;
const float *s = ep->dup + ((size_t)(Y >> 1) * (Wo >> 1) + (u >> 1)) * ep->dupC;
const int *tab = ep->dupt + ((Y & 1) * 2 + (u & 1));
for (int o = 0; o < Co; o++) t[o] += s[tab[o * 4]];
}
if (ep && ep->res) { const float *rr = ep->res + gp * Co; for (int o = 0; o < Co; o++) t[o] += rr[o]; }
memcpy(y + gp * Co, t, sizeof(float) * Co);
}
const int y1 = y0 + nb;
if (ep && ep->rel_x) {
size_t upto = y1 >= Ho ? xtot : (size_t)((y1 - 1) >> up) * W * C;
qiv_release(x, xtot, xrel, upto); if (upto > xrel) xrel = upto;
}
if (ep && ep->rel_scx && ep->scx) {
size_t tot = (size_t)Ho * Wo * scC, upto = (size_t)y1 * Wo * scC;
qiv_release((float *)ep->scx, tot, screl, upto); screl = upto;
}
if (ep && ep->rel_dup && ep->dup) {
size_t tot = (size_t)(Ho >> 1) * (Wo >> 1) * ep->dupC;
size_t upto = y1 >= Ho ? tot : (size_t)(y1 >> 1) * (Wo >> 1) * ep->dupC;
qiv_release((float *)ep->dup, tot, duprel, upto); duprel = upto;
}
}
free(P); free(Q); free(T); free(T2);
return 0;
}
/* One residual block. x [H][W][cin] is consumed unless keep: the result is
* written over x when the shapes allow it, otherwise into a new buffer and x is
* freed. Returns the output or NULL on out of memory (x is then still owned by
* the caller). */
static float *qiv_resnet(const QivRes *r, float *x, int H, int W, int keep){
const size_t HW = (size_t)H * W;
float *t = qiv_alloc(HW * r->cout);
if (!t) return NULL;
if (qiv_conv3(&r->c1, x, H, W, r->g1, 0, t, NULL)) { free(t); return NULL; }
QivEpi ep; memset(&ep, 0, sizeof ep);
ep.rel_x = 1; /* conv2 is the last reader of t */
float *y;
if (r->cin == r->cout) {
ep.res = x;
y = keep ? qiv_alloc(HW * r->cout) : x;
} else {
ep.scx = x; ep.sc = &r->sc; ep.rel_scx = !keep;
y = qiv_alloc(HW * r->cout);
}
if (!y || qiv_conv3(&r->c2, t, H, W, r->g2, 0, y, &ep)) { free(t); if (y && y != x) free(y); return NULL; }
free(t);
if (y != x && !keep) free(x);
return y;
}
/* Mid-block attention, in place: x += proj(softmax(q k^T / sqrt(C)) v), one head,
* every pixel attends to every pixel. Queries go in blocks so the score matrix
* stays small at 2048x2048 (16384 tokens). */
static int qiv_attention(const QiVae *v, float *x, int N){
const int C = v->qkv.cin;
const float scale = (float)sqrt((double)C);
float *xn = qiv_alloc((size_t)N * C), *qkv = qiv_alloc((size_t)N * 3 * C);
float *k = qiv_alloc((size_t)N * C), *vt = qiv_alloc((size_t)N * C);
int qb = (int)(qiv_attn_floats / (size_t)N); if (qb < 16) qb = 16; if (qb > N) qb = N;
float *S = qiv_alloc((size_t)qb * N);
int rc = -1;
if (!xn || !qkv || !k || !vt || !S) goto out;
#pragma omp parallel for schedule(static)
for (int p = 0; p < N; p++) qiv_norm_px(xn + (size_t)p * C, x + (size_t)p * C, C, v->attn_g, scale, 0);
qi_gemm(qkv, xn, N, &v->qkv.m, v->qkv.b);
#pragma omp parallel for schedule(static)
for (int p = 0; p < N; p++) {
memcpy(k + (size_t)p * C, qkv + (size_t)p * 3 * C + C, sizeof(float) * C);
for (int c = 0; c < C; c++) vt[(size_t)c * N + p] = qkv[(size_t)p * 3 * C + 2 * C + c];
}
const QiMat Km = { QI_F32, N, C, k, NULL }, Vm = { QI_F32, C, N, vt, NULL };
const float isd = 1.f / scale;
for (int i0 = 0; i0 < N; i0 += qb) {
int m = N - i0 < qb ? N - i0 : qb;
qi_gemm_ld(S, N, qkv + (size_t)i0 * 3 * C, 3 * C, m, &Km, NULL);
#pragma omp parallel for schedule(static)
for (int i = 0; i < m; i++) {
float *s = S + (size_t)i * N, mx = -INFINITY;
for (int j = 0; j < N; j++) { s[j] *= isd; if (s[j] > mx) mx = s[j]; }
double sum = 0;
for (int j = 0; j < N; j++) { s[j] = expf(s[j] - mx); sum += s[j]; }
float inv = (float)(1.0 / sum);
for (int j = 0; j < N; j++) s[j] *= inv;
}
qi_gemm(xn + (size_t)i0 * C, S, m, &Vm, NULL); /* xn is free again: reuse for O */
}
qi_gemm(qkv, xn, N, &v->proj.m, v->proj.b); /* qkv reused for proj(O) [N][C] */
#pragma omp parallel for schedule(static)
for (size_t i = 0; i < (size_t)N * C; i++) x[i] += qkv[i];
rc = 0;
out:
free(xn); free(qkv); free(k); free(vt); free(S);
return rc;
}
/* conv_out's [HW][OC] to the picture: torch.clamp(-1, 1), then postprocess
* ((x*0.5 + 0.5).clamp(0, 1), *255 in float32, numpy round (half to even), uint8;
* x*0.5 is exact, so an FMA here rounds exactly like torch's two ops). */
static void qiv_post(const float *o, size_t HW, int OC, uint8_t *rgba, float *out_f){
#pragma omp parallel for schedule(static)
for (size_t p = 0; p < HW; p++)
for (int c = 0; c < 4; c++) {
float f = c < OC ? o[p * OC + c] : 1.f;
f = f < -1.f ? -1.f : f > 1.f ? 1.f : f;
if (c < OC && out_f) out_f[(size_t)c * HW + p] = f;
float u = f * 0.5f + 0.5f;
u = u < 0.f ? 0.f : u > 1.f ? 1.f : u;
if (rgba) rgba[p * 4 + c] = (uint8_t)nearbyintf(u * 255.f);
}
}
/* z: normalized latents [h][w][z_dim] (the packed DiT output, token y*w + x).
* rgba: [scale*h][scale*w][4] uint8 top row first (alpha 255 if the model has 3
* output channels). out_f: optional [out_ch][scale*h][scale*w] in [-1, 1].
* Returns 0, or -1 on out of memory / bad arguments. */
static int qiv_decode(QiVae *v, const float *z, int h, int w, uint8_t *rgba, float *out_f){
if (!v || !z || h <= 0 || w <= 0) return -1;
const int prof = getenv("QIV_PROFILE") != NULL;
double t0 = qiv_now(), tl = t0;
const int Z = v->z_dim, steps = 3 + v->nblk;
int step = 0, H = h, W = w;
size_t HW = (size_t)H * W;
#define QIV_STAGE(name) do { double tn = qiv_now(); \
if (prof) fprintf(stderr, "[vae] %-10s %4dx%-4d %7.3f s\n", name, H, W, tn - tl); tl = tn; \
if (v->progress) v->progress(v->progress_ud, ++step, steps); } while (0)
float *a = qiv_alloc(HW * Z), *b = qiv_alloc(HW * Z), *x = NULL;
if (!a || !b) goto fail;
for (size_t p = 0; p < HW; p++)
for (int c = 0; c < Z; c++) a[p * Z + c] = z[p * Z + c] * v->std[c] + v->mean[c];
qi_gemm(b, a, (int)HW, &v->pqc.m, v->pqc.b);
free(a); a = NULL;
x = qiv_alloc(HW * v->conv_in.cout);
if (!x || qiv_conv3(&v->conv_in, b, H, W, NULL, 0, x, NULL)) goto fail;
free(b); b = NULL;
QIV_STAGE("conv_in");
float *nx;
if (!(nx = qiv_resnet(&v->mid[0], x, H, W, 0))) goto fail;
x = nx;
if (qiv_attention(v, x, (int)HW)) goto fail;
if (!(nx = qiv_resnet(&v->mid[1], x, H, W, 0))) goto fail;
x = nx;
QIV_STAGE("mid");
for (int i = 0; i < v->nblk; i++) {
const QivBlock *bk = &v->blk[i];
/* xin: kept for the DupUp3D shortcut of an upsampling block, so its
* first resnet must not write over it */
float *xin = bk->up ? x : NULL, *cur = x;
x = NULL;
for (int j = 0; j < bk->nres; j++) {
if (!(nx = qiv_resnet(&bk->res[j], cur, H, W, bk->up && j == 0))) {
if (cur != xin) free(cur);
free(xin); goto fail;
}
cur = nx;
}
if (bk->up) {
float *y = qiv_alloc(HW * 4 * bk->cout);
QivEpi ep; memset(&ep, 0, sizeof ep);
ep.dup = xin; ep.dupC = bk->cin; ep.dupt = bk->dup;
ep.rel_x = 1; ep.rel_dup = 1;
if (!y || qiv_conv3(&bk->upc, cur, H, W, NULL, 1, y, &ep)) {
free(y); free(cur); free(xin); goto fail;
}
free(cur); free(xin);
H *= 2; W *= 2; HW = (size_t)H * W;
x = y;
} else {
x = cur;
}
char nm[16]; snprintf(nm, sizeof nm, "block%d", i);
QIV_STAGE(nm);
}
{
const int OC = v->out_ch;
float *o = qiv_alloc(HW * OC);
QivEpi ep; memset(&ep, 0, sizeof ep); ep.rel_x = 1;
if (!o || qiv_conv3(&v->conv_out, x, H, W, v->g_out, 0, o, &ep)) { free(o); goto fail; }
free(x); x = NULL;
qiv_post(o, HW, OC, rgba, out_f);
free(o);
}
QIV_STAGE("conv_out");
if (prof) fprintf(stderr, "[vae] total %.3f s\n", qiv_now() - t0);
#undef QIV_STAGE
return 0;
fail:
free(a); free(b); free(x);
fprintf(stderr, "qwenimage vae: decode failed (out of memory?)\n");
return -1;
}
#endif