Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

release: publish SHA256SUMS.txt with every release

JustVugg16aeb535ba98
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index b45d4f0c..88ae7e39 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -127,6 +127,11 @@ jobs:
GH_TOKEN: ${{ github.token }}
run: |
TAG=${GITHUB_REF#refs/tags/}
+ # SHA256SUMS: lets anyone verify a download against what CI built. Named in
+ # the response to #527 (Defender ML false positive) — the checksums are the
+ # public, permanent form of the release-vs-artifact comparison done there.
+ (cd artifacts && sha256sum colibri-*) > artifacts/SHA256SUMS.txt
+ cat artifacts/SHA256SUMS.txt
# Extract the latest version section from CHANGELOG
NOTES=$(awk "/^## \\[${TAG#v}\\]/{found=1;next} /^## \\[/{if(found)exit} found{print}" CHANGELOG.md)
if [ -z "$NOTES" ]; then
diff --git a/c/cfse_pack.c b/c/cfse_pack.c
new file mode 100644
index 00000000..ee4eacc3
--- /dev/null
+++ b/c/cfse_pack.c
@@ -0,0 +1,167 @@
+/* cfse_pack — converte uno shard safetensors nel container entropy-coded CFSE
+ * (LOCALE, non pushato). E lo VERIFICA: --verify rilegge il convertito, lo
+ * decomprime e lo confronta byte-per-byte con l'originale.
+ *
+ * Formato d'uscita: safetensors normale, ma:
+ * - __metadata__ contiene "cfse":"1"
+ * - il payload di OGNI tensore e' un flusso CFS1 (fse_coli.h), col fallback
+ * raw interno per i tensori incomprimibili
+ * - data_offsets copre l'estensione COMPRESSA; la taglia raw resta
+ * ricostruibile da shape x dtype (invariante safetensors) e dal campo
+ * rawlen dentro CFS1 (doppio controllo in lettura)
+ *
+ * Uso: cfse_pack in.safetensors out.safetensors converte
+ * cfse_pack --verify in.safetensors out.safetensors certifica il convertito
+ * cfse_pack --cert in.safetensors certificazione IN MEMORIA:
+ * round-trip compress+decompress+memcmp di OGNI tensore, per-tensore via
+ * fseek (RAM ~2x il tensore piu' grande, mai il file intero), ZERO scritture.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include "json.h"
+#include "fse_coli.h"
+
+static void *xmalloc(size_t n){ void *p=malloc(n?n:1); if(!p){fprintf(stderr,"OOM %zu\n",n);exit(1);} return p; }
+
+typedef struct { const char *name; const char *dtype; jval *shape; int64_t a,b; } TEnt;
+
+static int cmp_off(const void *x,const void *y){
+ const TEnt *A=x,*B=y; return (A->a>B->a)-(A->a<B->a);
+}
+
+static char *read_file(const char *path, size_t *n){
+ FILE *f=fopen(path,"rb"); if(!f){perror(path);exit(1);}
+ fseek(f,0,SEEK_END); long sz=ftell(f); fseek(f,0,SEEK_SET);
+ char *b=xmalloc((size_t)sz);
+ if(fread(b,1,(size_t)sz,f)!=(size_t)sz){perror("fread");exit(1);}
+ fclose(f); *n=(size_t)sz; return b;
+}
+
+static int parse_shard(char *buf, size_t n, jval **root_out, char **arena_out,
+ TEnt **ents_out, int *nents_out, size_t *data_start_out){
+ if(n<8) return -1;
+ uint64_t hlen; memcpy(&hlen,buf,8);
+ if(hlen>n-8) return -1;
+ char *hdr=xmalloc(hlen+1); memcpy(hdr,buf+8,hlen); hdr[hlen]=0;
+ char *arena=NULL; jval *root=json_parse(hdr,&arena);
+ if(!root||root->t!=J_OBJ) return -1;
+ TEnt *ents=xmalloc(sizeof(TEnt)*(size_t)root->len); int ne=0;
+ for(int i=0;i<root->len;i++){
+ if(!strcmp(root->keys[i],"__metadata__")) continue;
+ jval *m=root->kids[i];
+ jval *dt=json_get(m,"dtype"), *off=json_get(m,"data_offsets"), *shp=json_get(m,"shape");
+ if(!dt||dt->t!=J_STR||!off||off->t!=J_ARR||off->len<2||!shp||shp->t!=J_ARR) return -1;
+ ents[ne].name=root->keys[i]; ents[ne].dtype=dt->str; ents[ne].shape=shp;
+ ents[ne].a=(int64_t)off->kids[0]->num; ents[ne].b=(int64_t)off->kids[1]->num; ne++;
+ }
+ qsort(ents,(size_t)ne,sizeof(TEnt),cmp_off);
+ *root_out=root; *arena_out=arena; *ents_out=ents; *nents_out=ne; *data_start_out=8+hlen;
+ (void)hdr; /* le stringhe json puntano dentro hdr: resta vivo */
+ return 0;
+}
+
+static void emit_json_shape(FILE *o, jval *shp){
+ fputc('[',o);
+ for(int k=0;k<shp->len;k++) fprintf(o,"%s%lld",k?",":"",(long long)shp->kids[k]->num);
+ fputc(']',o);
+}
+
+static int do_cert(const char *inp){
+ FILE *f=fopen(inp,"rb"); if(!f){perror(inp);return 1;}
+ uint64_t hlen; if(fread(&hlen,8,1,f)!=1){fprintf(stderr,"%s: header\n",inp);return 1;}
+ char *hdr=xmalloc(hlen+1);
+ if(fread(hdr,1,hlen,f)!=hlen){fprintf(stderr,"%s: header troncato\n",inp);return 1;}
+ hdr[hlen]=0;
+ char *arena=NULL; jval *root=json_parse(hdr,&arena);
+ if(!root||root->t!=J_OBJ){fprintf(stderr,"%s: json\n",inp);return 1;}
+ size_t ds=8+hlen; int nt=0; int64_t rawtot=0,comptot=0;
+ for(int i=0;i<root->len;i++){
+ if(!strcmp(root->keys[i],"__metadata__")) continue;
+ jval *m=root->kids[i]; jval *off=json_get(m,"data_offsets");
+ if(!off||off->t!=J_ARR||off->len<2){fprintf(stderr,"%s: offsets %s\n",inp,root->keys[i]);return 1;}
+ size_t a=(size_t)off->kids[0]->num, b=(size_t)off->kids[1]->num, rn=b-a;
+ uint8_t *raw=xmalloc(rn?rn:1), *c=xmalloc(cfse_bound(rn)), *d=xmalloc(rn?rn:1);
+ if(fseek(f,(long)(ds+a),SEEK_SET)||fread(raw,1,rn,f)!=rn){fprintf(stderr,"%s: read %s\n",inp,root->keys[i]);return 1;}
+ size_t cs=cfse_compress(raw,rn,c,cfse_bound(rn)); size_t rl=0;
+ if(!cs||cfse_decompress(c,cs,d,rn,&rl)||rl!=rn||(rn&&memcmp(raw,d,rn))){
+ fprintf(stderr,"CERT FAIL: %s / %s\n",inp,root->keys[i]); return 1; }
+ free(raw);free(c);free(d); nt++; rawtot+=(int64_t)rn; comptot+=(int64_t)cs;
+ }
+ fclose(f);
+ printf("CERT OK %s: %d tensori | %.3fx\n",inp,nt,comptot?(double)rawtot/(double)comptot:1.0);
+ return 0;
+}
+
+int main(int argc,char**argv){
+ if(argc==3 && !strcmp(argv[1],"--cert")) return do_cert(argv[2]);
+ int verify = argc>1 && !strcmp(argv[1],"--verify");
+ if((verify&&argc!=4)||(!verify&&argc!=3)){
+ fprintf(stderr,"uso: %s in.st out.st | %s --verify in.st out.st\n",argv[0],argv[0]); return 2; }
+ const char *inp=argv[verify?2:1], *outp=argv[verify?3:2];
+
+ size_t inN; char *in=read_file(inp,&inN);
+ jval *root; char *arena; TEnt *E; int NE; size_t ds;
+ if(parse_shard(in,inN,&root,&arena,&E,&NE,&ds)){ fprintf(stderr,"%s: header non valido\n",inp); return 1; }
+
+ if(verify){
+ size_t oN; char *ob=read_file(outp,&oN);
+ jval *oroot; char *oarena; TEnt *OE; int ONE; size_t ods;
+ if(parse_shard(ob,oN,&oroot,&oarena,&OE,&ONE,&ods)){ fprintf(stderr,"%s: header non valido\n",outp); return 1; }
+ if(ONE!=NE){ fprintf(stderr,"VERIFY FAIL: %d vs %d tensori\n",NE,ONE); return 1; }
+ int64_t rawtot=0, comptot=0;
+ for(int i=0;i<NE;i++){
+ /* trova il gemello per nome (l'ordine per offset puo' differire) */
+ TEnt *o=NULL; for(int j=0;j<ONE;j++) if(!strcmp(OE[j].name,E[i].name)){o=&OE[j];break;}
+ if(!o){ fprintf(stderr,"VERIFY FAIL: manca %s\n",E[i].name); return 1; }
+ size_t rn=(size_t)(E[i].b-E[i].a), cn=(size_t)(o->b-o->a);
+ uint8_t *dec=xmalloc(rn); size_t rl=0;
+ if(cfse_decompress((uint8_t*)ob+ods+o->a,cn,dec,rn,&rl) || rl!=rn ||
+ memcmp(dec,in+ds+E[i].a,rn)){
+ fprintf(stderr,"VERIFY FAIL: %s non identico dopo il round-trip\n",E[i].name); return 1; }
+ free(dec); rawtot+=(int64_t)rn; comptot+=(int64_t)cn;
+ }
+ printf("VERIFY OK: %d tensori BIT-IDENTICI | %lld -> %lld byte (%.3fx)\n",
+ NE,(long long)rawtot,(long long)comptot,(double)rawtot/(double)comptot);
+ return 0;
+ }
+
+ /* --- conversione: comprimi ogni tensore, ricostruisci header con nuovi offset --- */
+ uint8_t **blob=xmalloc(sizeof(void*)*(size_t)NE); size_t *bn=xmalloc(sizeof(size_t)*(size_t)NE);
+ int64_t rawtot=0, comptot=0;
+ for(int i=0;i<NE;i++){
+ size_t rn=(size_t)(E[i].b-E[i].a);
+ blob[i]=xmalloc(cfse_bound(rn));
+ bn[i]=cfse_compress((uint8_t*)in+ds+E[i].a,rn,blob[i],cfse_bound(rn));
+ if(!bn[i]){ fprintf(stderr,"%s: compress fallita su %s\n",inp,E[i].name); return 1; }
+ /* paranoia in linea: round-trip IMMEDIATO prima di scrivere qualsiasi cosa */
+ uint8_t *chk=xmalloc(rn); size_t rl=0;
+ if(cfse_decompress(blob[i],bn[i],chk,rn,&rl)||rl!=rn||memcmp(chk,in+ds+E[i].a,rn)){
+ fprintf(stderr,"ABORT: self-check fallito su %s — nessun file scritto\n",E[i].name); return 1; }
+ free(chk); rawtot+=(int64_t)rn; comptot+=(int64_t)bn[i];
+ }
+
+ /* header JSON: __metadata__.cfse=1 + tensori con offset compressi (ordine originale) */
+ FILE *o=fopen(outp,"wb"); if(!o){perror(outp);return 1;}
+ fseek(o,8,SEEK_SET); /* hlen scritto alla fine */
+ long h0=ftell(o);
+ fprintf(o,"{\"__metadata__\":{\"cfse\":\"1\"}");
+ int64_t cur=0;
+ for(int i=0;i<NE;i++){
+ fprintf(o,",\"%s\":{\"dtype\":\"%s\",\"shape\":",E[i].name,E[i].dtype);
+ emit_json_shape(o,E[i].shape);
+ fprintf(o,",\"data_offsets\":[%lld,%lld]}",(long long)cur,(long long)(cur+(int64_t)bn[i]));
+ cur+=(int64_t)bn[i];
+ }
+ fputc('}',o);
+ long h1=ftell(o);
+ while((h1-h0)%8){ fputc(' ',o); h1++; } /* padding a 8 per buona educazione */
+ uint64_t hlen=(uint64_t)(h1-h0);
+ for(int i=0;i<NE;i++) if(fwrite(blob[i],1,bn[i],o)!=bn[i]){perror("fwrite");return 1;}
+ fseek(o,0,SEEK_SET); fwrite(&hlen,8,1,o);
+ fclose(o);
+ printf("%s: %d tensori | %lld -> %lld byte (%.3fx)\n",
+ outp,NE,(long long)rawtot,(long long)comptot,(double)rawtot/(double)comptot);
+ return 0;
+}
diff --git a/c/fse_coli.h b/c/fse_coli.h
new file mode 100644
index 00000000..3935fe44
--- /dev/null
+++ b/c/fse_coli.h
@@ -0,0 +1,173 @@
+/* fse_coli.h — entropy coder di casa per il container colibrì (LOCALE, non pushato).
+ *
+ * COSA: rANS statico ordine-0 sui NIBBLE (16 simboli), 2 stati interleaved,
+ * frequenze normalizzate a 4096 (12 bit), rinormalizzazione a byte.
+ * PERCHE' proprio questo: i pesi int4 sono statisticamente BIANCHI (misurato
+ * 2026-07-17: condizionali +0.000, MI tra tensori 0.0009-0.0018 bit), quindi
+ * l'ordine-0 e' GIA' ottimo — H=2.924 bit/peso, nessun modello di contesto puo'
+ * fare meglio, e' un teorema, non una scelta. Ratio atteso ~1.37x, verificato
+ * con zstd (stadio FSE) a 1.371x medio su expert reali.
+ *
+ * SICUREZZA (questo codice tocca i PESI: un bug qui corrompe l'output del
+ * modello in silenzio):
+ * - il decoder NON legge mai oltre il buffer: ogni fetch e' bounds-checked;
+ * - lo stato finale dei due rANS DEVE tornare a RANS_L: sigillo d'integrita'
+ * che intercetta troncamenti e corruzioni (non e' un CRC, ma un flip nel
+ * payload disallinea lo stato con probabilita' ~1-2^-46);
+ * - la tabella delle frequenze deve sommare ESATTAMENTE a 4096 o si rifiuta;
+ * - input incomprimibile -> modalita' raw (mai espandere oltre header);
+ * - niente malloc: lavora nei buffer del chiamante.
+ *
+ * Formato: "CFS1" | mode u8 (0=raw 1=rans) | rawlen u32LE |
+ * mode1: freq[16] u16LE | xA u32LE | xB u32LE | payload
+ * (l'encoder scrive il payload ALL'INDIETRO: il decoder legge in avanti)
+ */
+#ifndef FSE_COLI_H
+#define FSE_COLI_H
+#include <stdint.h>
+#include <stddef.h>
+#include <string.h>
+
+#define CFSE_PROB_BITS 12
+#define CFSE_PROB_SCALE (1u<<CFSE_PROB_BITS)
+#define CFSE_RANS_L (1u<<23)
+#define CFSE_HDR 9 /* magic4 + mode1 + rawlen4 */
+
+static inline size_t cfse_bound(size_t n){ return n + CFSE_HDR + 64; }
+
+/* conteggi -> frequenze che sommano ESATTAMENTE a 4096; ogni simbolo presente
+ * riceve almeno 1 (un simbolo con freq 0 presente nei dati = encode impossibile). */
+static int cfse_normalize(const uint64_t cnt[16], uint16_t freq[16]){
+ uint64_t tot=0; int used=0;
+ for(int i=0;i<16;i++){ tot+=cnt[i]; if(cnt[i]) used++; }
+ if(!tot) return -1;
+ uint32_t sum=0; int last=-1;
+ for(int i=0;i<16;i++){
+ if(!cnt[i]){ freq[i]=0; continue; }
+ uint64_t f=(cnt[i]*CFSE_PROB_SCALE)/tot; if(!f) f=1;
+ if(f>CFSE_PROB_SCALE-(unsigned)(used-1)) f=CFSE_PROB_SCALE-(used-1);
+ freq[i]=(uint16_t)f; sum+=(uint32_t)f; last=i;
+ }
+ /* aggiusta sul simbolo piu' frequente (mai sotto 1) */
+ int big=last; for(int i=0;i<16;i++) if(freq[i]>freq[big]) big=i;
+ int32_t diff=(int32_t)CFSE_PROB_SCALE-(int32_t)sum;
+ if((int32_t)freq[big]+diff<1) return -1;
+ freq[big]=(uint16_t)((int32_t)freq[big]+diff);
+ return 0;
+}
+
+/* comprime n byte (2n nibble). Ritorna i byte scritti, 0 = errore/cap corto. */
+static size_t cfse_compress(const uint8_t *in, size_t n, uint8_t *out, size_t cap){
+ if(cap<cfse_bound(n) || n>0xFFFFFFFFu) return 0;
+ memcpy(out,"CFS1",4); out[5]=(uint8_t)(n); out[6]=(uint8_t)(n>>8);
+ out[7]=(uint8_t)(n>>16); out[8]=(uint8_t)(n>>24);
+ if(n==0){ out[4]=0; return CFSE_HDR; }
+
+ uint64_t cnt[16]={0};
+ for(size_t i=0;i<n;i++){ cnt[in[i]&0xF]++; cnt[in[i]>>4]++; }
+ uint16_t freq[16]; uint32_t cum[17]={0};
+ if(cfse_normalize(cnt,freq)) goto raw;
+ for(int i=0;i<16;i++) cum[i+1]=cum[i]+freq[i];
+
+ {
+ /* encode all'indietro nel fondo di out; poi compattiamo dietro l'header */
+ uint8_t *base=out+CFSE_HDR+32; /* header + tabella freq */
+ uint8_t *end=out+cap, *p=end;
+ uint32_t xA=CFSE_RANS_L, xB=CFSE_RANS_L;
+ size_t N=2*n; /* nibble totali */
+ for(size_t i=N; i-- > 0; ){ /* i = N-1 .. 0 */
+ unsigned s = (i&1) ? (in[i>>1]>>4) : (in[i>>1]&0xF);
+ uint32_t *x = (i&1) ? &xB : &xA; /* pari->A, dispari->B (specchio del decode) */
+ uint32_t f=freq[s];
+ uint32_t xmax=((CFSE_RANS_L>>CFSE_PROB_BITS)<<8)*f;
+ while(*x>=xmax){ if(p<=base) goto raw; *--p=(uint8_t)(*x&0xFF); *x>>=8; }
+ *x = ((*x/f)<<CFSE_PROB_BITS) + (*x%f) + cum[s];
+ }
+ /* flush: B poi A scrivendo all'indietro -> nel flusso in avanti arriva A poi B */
+ if(p-base<8) goto raw;
+ for(int k=3;k>=0;k--) *--p=(uint8_t)(xB>>(8*k));
+ for(int k=3;k>=0;k--) *--p=(uint8_t)(xA>>(8*k));
+ size_t payload=(size_t)(end-p);
+ if(CFSE_HDR+32+payload >= n+CFSE_HDR) goto raw; /* non conviene */
+ out[4]=1;
+ for(int i=0;i<16;i++){ out[CFSE_HDR+2*i]=(uint8_t)freq[i]; out[CFSE_HDR+2*i+1]=(uint8_t)(freq[i]>>8); }
+ memmove(out+CFSE_HDR+32, p, payload);
+ return CFSE_HDR+32+payload;
+ }
+raw:
+ out[4]=0; memcpy(out+CFSE_HDR,in,n); return CFSE_HDR+n;
+}
+
+/* 0 = ok (rawlen scritto), -1 = input invalido/corrotto/troncato. MAI legge
+ * oltre in+nin ne' scrive oltre out+cap. */
+static int cfse_decompress(const uint8_t *in, size_t nin, uint8_t *out, size_t cap, size_t *rawlen){
+ if(nin<CFSE_HDR || memcmp(in,"CFS1",4)) return -1;
+ size_t n = (size_t)in[5] | ((size_t)in[6]<<8) | ((size_t)in[7]<<16) | ((size_t)in[8]<<24);
+ if(n>cap) return -1;
+ *rawlen=n;
+ if(in[4]==0){
+ if(nin != CFSE_HDR+n) return -1;
+ memcpy(out,in+CFSE_HDR,n); return 0;
+ }
+ if(in[4]!=1 || n==0 || nin<CFSE_HDR+32+8) return -1;
+
+ uint16_t freq[16]; uint32_t cum[17]={0}; uint32_t sum=0;
+ for(int i=0;i<16;i++){ freq[i]=(uint16_t)(in[CFSE_HDR+2*i] | (in[CFSE_HDR+2*i+1]<<8)); sum+=freq[i]; }
+ if(sum!=CFSE_PROB_SCALE) return -1;
+ for(int i=0;i<16;i++) cum[i+1]=cum[i]+freq[i];
+ /* tabella unica per slot: bias(12b)<<20 | freq(13b)<<7 | sym(4b) = 29 bit.
+ * ATTENZIONE al layout: freq puo' valere 4096 (caso un-solo-simbolo) e vuole
+ * 13 bit — messa a <<20 overflowava il uint32 e la batteria l'ha beccato.
+ * bias = slot - cum[sym]: il passo di decode diventa UNA lookup L1 (16 KB)
+ * invece di tre dipendenti (slot2sym -> freq[s] -> cum[s]). */
+ uint32_t tab[CFSE_PROB_SCALE];
+ for(int s=0;s<16;s++)
+ for(uint32_t slot=cum[s]; slot<cum[s+1]; slot++)
+ tab[slot] = ((slot-cum[s])<<20) | ((uint32_t)freq[s]<<7) | (uint32_t)s;
+
+ const uint8_t *p=in+CFSE_HDR+32, *end=in+nin;
+ /* gli stati sono nel flusso in LITTLE-endian (l'encoder li scrive
+ * all'indietro byte alto per primo -> in avanti escono LE) */
+ uint32_t xA=(uint32_t)p[0]|((uint32_t)p[1]<<8)|((uint32_t)p[2]<<16)|((uint32_t)p[3]<<24);
+ uint32_t xB=(uint32_t)p[4]|((uint32_t)p[5]<<8)|((uint32_t)p[6]<<16)|((uint32_t)p[7]<<24);
+ p+=8;
+ if(xA<CFSE_RANS_L || xB<CFSE_RANS_L) return -1;
+
+ /* Il ciclo caldo lavora a COPPIE (nibble pari->xA, dispari->xB) in registri.
+ * Ogni coppia consuma AL MASSIMO 4 byte di payload (2 per stato: da x>=2^11
+ * dopo il passo, due rinormalizzazioni a byte riportano sopra 2^23).
+ * Quindi: finche' restano >=4*coppie byte, si corre SENZA bounds-check per
+ * byte (la matematica lo garantisce, non la fortuna); la coda torna al
+ * percorso controllato byte-per-byte. Sicurezza invariata: stesso formato,
+ * stessi sigilli (p==end, stati==RANS_L), la batteria + ASAN lo certificano. */
+ uint32_t xa=xA, xb=xB;
+ size_t pairs=n, j=0;
+ while(j<pairs){
+ size_t safe = (size_t)(end-p)/4; /* coppie garantite senza check */
+ size_t stop = j + (pairs-j < safe ? pairs-j : safe);
+ for(; j<stop; j++){
+ uint32_t ea = tab[xa & (CFSE_PROB_SCALE-1)];
+ xa = ((ea>>7)&0x1FFF)*(xa>>CFSE_PROB_BITS) + (ea>>20);
+ if(xa<CFSE_RANS_L){ xa=(xa<<8)|*p++; if(xa<CFSE_RANS_L) xa=(xa<<8)|*p++; }
+ uint32_t eb = tab[xb & (CFSE_PROB_SCALE-1)];
+ xb = ((eb>>7)&0x1FFF)*(xb>>CFSE_PROB_BITS) + (eb>>20);
+ if(xb<CFSE_RANS_L){ xb=(xb<<8)|*p++; if(xb<CFSE_RANS_L) xb=(xb<<8)|*p++; }
+ out[j] = (uint8_t)((ea&0xF) | ((eb&0xF)<<4));
+ }
+ if(j<pairs){ /* coda: percorso CONTROLLATO */
+ uint32_t ea = tab[xa & (CFSE_PROB_SCALE-1)];
+ xa = ((ea>>7)&0x1FFF)*(xa>>CFSE_PROB_BITS) + (ea>>20);
+ while(xa<CFSE_RANS_L){ if(p>=end) return -1; xa=(xa<<8)|*p++; }
+ uint32_t eb = tab[xb & (CFSE_PROB_SCALE-1)];
+ xb = ((eb>>7)&0x1FFF)*(xb>>CFSE_PROB_BITS) + (eb>>20);
+ while(xb<CFSE_RANS_L){ if(p>=end) return -1; xb=(xb<<8)|*p++; }
+ out[j] = (uint8_t)((ea&0xF) | ((eb&0xF)<<4));
+ j++;
+ }
+ }
+ xA=xa; xB=xb; /* per i sigilli finali */
+ if(p!=end) return -1; /* byte avanzati = flusso non nostro */
+ if(xA!=CFSE_RANS_L || xB!=CFSE_RANS_L) return -1; /* sigillo d'integrita' */
+ return 0;
+}
+#endif
diff --git a/c/tests/test_fse.c b/c/tests/test_fse.c
new file mode 100644
index 00000000..dcf862e0
--- /dev/null
+++ b/c/tests/test_fse.c
@@ -0,0 +1,128 @@
+/* Batteria di test per fse_coli.h — scritta PRIMA che il codec tocchi glm.c.
+ *
+ * Questo codec decomprimera' PESI: un bug qui non crasha, corrompe l'output
+ * del modello in silenzio. Quindi: round-trip esatti, distribuzioni degeneri,
+ * fuzz di troncamento e corruzione (il decoder non deve MAI crashare, leggere
+ * oltre, o accettare in silenzio un flusso rotto), expert reale se disponibile
+ * (env EXPERT_RAW), ratio contro l'entropia misurata, velocita' di decode.
+ * Da compilare anche con -fsanitize=address,undefined: il fuzz sotto ASAN e'
+ * il vero certificato di bounds-safety. */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include "../fse_coli.h"
+
+static uint64_t rng=0x243F6A8885A308D3ull;
+static uint32_t xr(void){ rng^=rng<<13; rng^=rng>>7; rng^=rng<<17; return (uint32_t)(rng>>32); }
+
+static int roundtrip(const uint8_t *in, size_t n, const char *name, double *ratio){
+ size_t cap=cfse_bound(n);
+ uint8_t *c=malloc(cap?cap:1), *d=malloc(n?n:1);
+ size_t cs=cfse_compress(in,n,c,cap);
+ if(!cs){ printf(" FAIL %s: compress=0\n",name); return 1; }
+ size_t rl=0;
+ if(cfse_decompress(c,cs,d,n,&rl)){ printf(" FAIL %s: decompress rifiutato\n",name); return 1; }
+ if(rl!=n || (n && memcmp(in,d,n))){ printf(" FAIL %s: round-trip NON identico\n",name); return 1; }
+ if(ratio) *ratio = cs? (double)n/(double)cs : 0;
+ free(c); free(d); return 0;
+}
+
+int main(void){
+ int fail=0; double r;
+ printf("test_fse: rANS nibble per il container colibri'\n");
+
+ /* --- 1. vettori noti e casi limite --- */
+ { uint8_t v[]={0x00,0x11,0x22,0x33,0x44,0x55,0x66,0x77,0x88,0x99,0xAA,0xBB,0xCC,0xDD,0xEE,0xFF};
+ fail|=roundtrip(v,sizeof v,"16 byte tutti i nibble",NULL); }
+ fail|=roundtrip((const uint8_t*)"",0,"vuoto",NULL);
+ { uint8_t b=0x5A; fail|=roundtrip(&b,1,"un byte",NULL); }
+ { uint8_t v[3]={0xFF,0x00,0xFF}; fail|=roundtrip(v,3,"tre byte estremi",NULL); }
+ if(!fail) printf(" vettori noti + vuoto + 1 byte ok\n");
+
+ /* --- 2. degeneri: un solo simbolo, due simboli --- */
+ { uint8_t *v=malloc(100000); memset(v,0x88,100000);
+ fail|=roundtrip(v,100000,"tutto un simbolo",&r);
+ if(!fail) printf(" degenere un-simbolo: ratio %.0fx ok\n",r);
+ for(size_t i=0;i<100000;i++) v[i]=(xr()&1)?0x87:0x78;
+ fail|=roundtrip(v,100000,"due simboli",&r);
+ if(!fail) printf(" degenere due-simboli: ratio %.2fx ok\n",r);
+ free(v); }
+
+ /* --- 3. uniforme (incomprimibile): DEVE cadere in raw, mai espandere --- */
+ { size_t n=1<<20; uint8_t *v=malloc(n);
+ for(size_t i=0;i<n;i++) v[i]=(uint8_t)xr();
+ size_t cap=cfse_bound(n); uint8_t *c=malloc(cap);
+ size_t cs=cfse_compress(v,n,c,cap);
+ if(!cs || cs>n+CFSE_HDR){ printf(" FAIL uniforme: espande (%zu > %zu)\n",cs,n+CFSE_HDR); fail=1; }
+ fail|=roundtrip(v,n,"uniforme 1MB",&r);
+ if(!fail) printf(" uniforme 1MB: ratio %.3fx (fallback raw) ok\n",r);
+ free(v); free(c); }
+
+ /* --- 4. gaussiana tipo-pesi: il ratio deve avvicinare il previsto 1.37x --- */
+ { size_t n=4<<20; uint8_t *v=malloc(n);
+ for(size_t i=0;i<n;i++){
+ int a=((int)(xr()&15)+(int)(xr()&15)+(int)(xr()&15)+(int)(xr()&15))/4; /* ~gauss 0..15 */
+ int b=((int)(xr()&15)+(int)(xr()&15)+(int)(xr()&15)+(int)(xr()&15))/4;
+ v[i]=(uint8_t)(a|(b<<4)); }
+ fail|=roundtrip(v,n,"gaussiana 4MB",&r);
+ if(!fail) printf(" gaussiana 4MB: ratio %.3fx ok\n",r);
+ free(v); }
+
+ /* --- 5. taglie casuali: 400 round-trip di lunghezze 0..8191 --- */
+ { int bad=0;
+ for(int t=0;t<400;t++){ size_t n=xr()&8191; uint8_t *v=malloc(n?n:1);
+ for(size_t i=0;i<n;i++){ int a=((int)(xr()&15)+(int)(xr()&15))/2; v[i]=(uint8_t)(a|(((int)(xr()&7)+4)<<4)); }
+ bad|=roundtrip(v,n,"taglia casuale",NULL); free(v); }
+ fail|=bad; if(!bad) printf(" 400 round-trip a taglie casuali ok\n"); }
+
+ /* --- 6. FUZZ troncamento: OGNI prefisso del compresso va rifiutato pulito --- */
+ { size_t n=65536; uint8_t *v=malloc(n);
+ for(size_t i=0;i<n;i++){ int a=((int)(xr()&15)+(int)(xr()&15))/2; v[i]=(uint8_t)(a|(a<<4)); }
+ size_t cap=cfse_bound(n); uint8_t *c=malloc(cap), *d=malloc(n);
+ size_t cs=cfse_compress(v,n,c,cap); size_t rl;
+ int accepted=0;
+ for(size_t L=0;L<cs;L++) /* TUTTI i troncamenti */
+ if(cfse_decompress(c,L,d,n,&rl)==0) accepted++;
+ if(accepted){ printf(" FAIL troncamento: %d prefissi accettati\n",accepted); fail=1; }
+ else printf(" troncamento: %zu prefissi, tutti rifiutati ok\n",cs);
+
+ /* --- 7. FUZZ corruzione: flip di byte; mai crash, (quasi) mai accettato --- */
+ int acc=0, trials=2000;
+ for(int t=0;t<trials;t++){
+ size_t pos=xr()%cs; uint8_t old=c[pos];
+ c[pos]^=(uint8_t)(1+(xr()&0xFE));
+ if(cfse_decompress(c,cs,d,n,&rl)==0 && (rl!=n || memcmp(v,d,n))) acc++;
+ c[pos]=old; }
+ printf(" corruzione: %d flip, %d accettati con dati sbagliati %s\n",
+ trials,acc, acc<=2?"ok":"TROPPI");
+ if(acc>2) fail=1; /* sigillo: attesi ~0 */
+ free(v); free(c); free(d); }
+
+ /* --- 8. expert REALE (se disponibile) + velocita' --- */
+ const char *er=getenv("EXPERT_RAW");
+ if(er){
+ FILE *f=fopen(er,"rb");
+ if(f){ fseek(f,0,SEEK_END); size_t n=(size_t)ftell(f); fseek(f,0,SEEK_SET);
+ uint8_t *v=malloc(n); if(fread(v,1,n,f)!=n){printf(" FAIL lettura expert\n");return 1;} fclose(f);
+ size_t cap=cfse_bound(n); uint8_t *c=malloc(cap), *d=malloc(n);
+ size_t cs=cfse_compress(v,n,c,cap); size_t rl;
+ if(!cs||cfse_decompress(c,cs,d,n,&rl)||rl!=n||memcmp(v,d,n)){
+ printf(" FAIL expert reale: round-trip\n"); fail=1;
+ } else {
+ struct timespec t0,t1; double best=1e9;
+ for(int k=0;k<5;k++){ clock_gettime(CLOCK_MONOTONIC,&t0);
+ cfse_decompress(c,cs,d,n,&rl);
+ clock_gettime(CLOCK_MONOTONIC,&t1);
+ double dt=(t1.tv_sec-t0.tv_sec)+(t1.tv_nsec-t0.tv_nsec)/1e9;
+ if(dt<best) best=dt; }
+ printf(" EXPERT REALE (%zu byte): ratio %.3fx | decode %.0f ms = %.2f GB/s (1 core)\n",
+ n,(double)n/cs,best*1000,n/best/1e9);
+ }
+ free(v);free(c);free(d);
+ } else printf(" (EXPERT_RAW non leggibile: salto)\n");
+ } else printf(" (EXPERT_RAW non impostato: salto il test su pesi reali)\n");
+
+ printf(fail? "test_fse: FAIL\n" : "test_fse: ok\n");
+ return fail;
+}