Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
release: publish SHA256SUMS.txt with every release
JustVugg16aeb535ba98
diff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex b45d4f0c..88ae7e39 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -127,6 +127,11 @@ jobs:GH_TOKEN: ${{ github.token }}run: |TAG=${GITHUB_REF#refs/tags/}+ # SHA256SUMS: lets anyone verify a download against what CI built. Named in+ # the response to #527 (Defender ML false positive) — the checksums are the+ # public, permanent form of the release-vs-artifact comparison done there.+ (cd artifacts && sha256sum colibri-*) > artifacts/SHA256SUMS.txt+ cat artifacts/SHA256SUMS.txt# Extract the latest version section from CHANGELOGNOTES=$(awk "/^## \\[${TAG#v}\\]/{found=1;next} /^## \\[/{if(found)exit} found{print}" CHANGELOG.md)if [ -z "$NOTES" ]; thendiff --git a/c/cfse_pack.c b/c/cfse_pack.cnew file mode 100644index 00000000..ee4eacc3--- /dev/null+++ b/c/cfse_pack.c@@ -0,0 +1,167 @@+/* cfse_pack — converte uno shard safetensors nel container entropy-coded CFSE+ * (LOCALE, non pushato). E lo VERIFICA: --verify rilegge il convertito, lo+ * decomprime e lo confronta byte-per-byte con l'originale.+ *+ * Formato d'uscita: safetensors normale, ma:+ * - __metadata__ contiene "cfse":"1"+ * - il payload di OGNI tensore e' un flusso CFS1 (fse_coli.h), col fallback+ * raw interno per i tensori incomprimibili+ * - data_offsets copre l'estensione COMPRESSA; la taglia raw resta+ * ricostruibile da shape x dtype (invariante safetensors) e dal campo+ * rawlen dentro CFS1 (doppio controllo in lettura)+ *+ * Uso: cfse_pack in.safetensors out.safetensors converte+ * cfse_pack --verify in.safetensors out.safetensors certifica il convertito+ * cfse_pack --cert in.safetensors certificazione IN MEMORIA:+ * round-trip compress+decompress+memcmp di OGNI tensore, per-tensore via+ * fseek (RAM ~2x il tensore piu' grande, mai il file intero), ZERO scritture.+ */+#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <stdint.h>+#include "json.h"+#include "fse_coli.h"++static void *xmalloc(size_t n){ void *p=malloc(n?n:1); if(!p){fprintf(stderr,"OOM %zu\n",n);exit(1);} return p; }++typedef struct { const char *name; const char *dtype; jval *shape; int64_t a,b; } TEnt;++static int cmp_off(const void *x,const void *y){+ const TEnt *A=x,*B=y; return (A->a>B->a)-(A->a<B->a);+}++static char *read_file(const char *path, size_t *n){+ FILE *f=fopen(path,"rb"); if(!f){perror(path);exit(1);}+ fseek(f,0,SEEK_END); long sz=ftell(f); fseek(f,0,SEEK_SET);+ char *b=xmalloc((size_t)sz);+ if(fread(b,1,(size_t)sz,f)!=(size_t)sz){perror("fread");exit(1);}+ fclose(f); *n=(size_t)sz; return b;+}++static int parse_shard(char *buf, size_t n, jval **root_out, char **arena_out,+ TEnt **ents_out, int *nents_out, size_t *data_start_out){+ if(n<8) return -1;+ uint64_t hlen; memcpy(&hlen,buf,8);+ if(hlen>n-8) return -1;+ char *hdr=xmalloc(hlen+1); memcpy(hdr,buf+8,hlen); hdr[hlen]=0;+ char *arena=NULL; jval *root=json_parse(hdr,&arena);+ if(!root||root->t!=J_OBJ) return -1;+ TEnt *ents=xmalloc(sizeof(TEnt)*(size_t)root->len); int ne=0;+ for(int i=0;i<root->len;i++){+ if(!strcmp(root->keys[i],"__metadata__")) continue;+ jval *m=root->kids[i];+ jval *dt=json_get(m,"dtype"), *off=json_get(m,"data_offsets"), *shp=json_get(m,"shape");+ if(!dt||dt->t!=J_STR||!off||off->t!=J_ARR||off->len<2||!shp||shp->t!=J_ARR) return -1;+ ents[ne].name=root->keys[i]; ents[ne].dtype=dt->str; ents[ne].shape=shp;+ ents[ne].a=(int64_t)off->kids[0]->num; ents[ne].b=(int64_t)off->kids[1]->num; ne++;+ }+ qsort(ents,(size_t)ne,sizeof(TEnt),cmp_off);+ *root_out=root; *arena_out=arena; *ents_out=ents; *nents_out=ne; *data_start_out=8+hlen;+ (void)hdr; /* le stringhe json puntano dentro hdr: resta vivo */+ return 0;+}++static void emit_json_shape(FILE *o, jval *shp){+ fputc('[',o);+ for(int k=0;k<shp->len;k++) fprintf(o,"%s%lld",k?",":"",(long long)shp->kids[k]->num);+ fputc(']',o);+}++static int do_cert(const char *inp){+ FILE *f=fopen(inp,"rb"); if(!f){perror(inp);return 1;}+ uint64_t hlen; if(fread(&hlen,8,1,f)!=1){fprintf(stderr,"%s: header\n",inp);return 1;}+ char *hdr=xmalloc(hlen+1);+ if(fread(hdr,1,hlen,f)!=hlen){fprintf(stderr,"%s: header troncato\n",inp);return 1;}+ hdr[hlen]=0;+ char *arena=NULL; jval *root=json_parse(hdr,&arena);+ if(!root||root->t!=J_OBJ){fprintf(stderr,"%s: json\n",inp);return 1;}+ size_t ds=8+hlen; int nt=0; int64_t rawtot=0,comptot=0;+ for(int i=0;i<root->len;i++){+ if(!strcmp(root->keys[i],"__metadata__")) continue;+ jval *m=root->kids[i]; jval *off=json_get(m,"data_offsets");+ if(!off||off->t!=J_ARR||off->len<2){fprintf(stderr,"%s: offsets %s\n",inp,root->keys[i]);return 1;}+ size_t a=(size_t)off->kids[0]->num, b=(size_t)off->kids[1]->num, rn=b-a;+ uint8_t *raw=xmalloc(rn?rn:1), *c=xmalloc(cfse_bound(rn)), *d=xmalloc(rn?rn:1);+ if(fseek(f,(long)(ds+a),SEEK_SET)||fread(raw,1,rn,f)!=rn){fprintf(stderr,"%s: read %s\n",inp,root->keys[i]);return 1;}+ size_t cs=cfse_compress(raw,rn,c,cfse_bound(rn)); size_t rl=0;+ if(!cs||cfse_decompress(c,cs,d,rn,&rl)||rl!=rn||(rn&&memcmp(raw,d,rn))){+ fprintf(stderr,"CERT FAIL: %s / %s\n",inp,root->keys[i]); return 1; }+ free(raw);free(c);free(d); nt++; rawtot+=(int64_t)rn; comptot+=(int64_t)cs;+ }+ fclose(f);+ printf("CERT OK %s: %d tensori | %.3fx\n",inp,nt,comptot?(double)rawtot/(double)comptot:1.0);+ return 0;+}++int main(int argc,char**argv){+ if(argc==3 && !strcmp(argv[1],"--cert")) return do_cert(argv[2]);+ int verify = argc>1 && !strcmp(argv[1],"--verify");+ if((verify&&argc!=4)||(!verify&&argc!=3)){+ fprintf(stderr,"uso: %s in.st out.st | %s --verify in.st out.st\n",argv[0],argv[0]); return 2; }+ const char *inp=argv[verify?2:1], *outp=argv[verify?3:2];++ size_t inN; char *in=read_file(inp,&inN);+ jval *root; char *arena; TEnt *E; int NE; size_t ds;+ if(parse_shard(in,inN,&root,&arena,&E,&NE,&ds)){ fprintf(stderr,"%s: header non valido\n",inp); return 1; }++ if(verify){+ size_t oN; char *ob=read_file(outp,&oN);+ jval *oroot; char *oarena; TEnt *OE; int ONE; size_t ods;+ if(parse_shard(ob,oN,&oroot,&oarena,&OE,&ONE,&ods)){ fprintf(stderr,"%s: header non valido\n",outp); return 1; }+ if(ONE!=NE){ fprintf(stderr,"VERIFY FAIL: %d vs %d tensori\n",NE,ONE); return 1; }+ int64_t rawtot=0, comptot=0;+ for(int i=0;i<NE;i++){+ /* trova il gemello per nome (l'ordine per offset puo' differire) */+ TEnt *o=NULL; for(int j=0;j<ONE;j++) if(!strcmp(OE[j].name,E[i].name)){o=&OE[j];break;}+ if(!o){ fprintf(stderr,"VERIFY FAIL: manca %s\n",E[i].name); return 1; }+ size_t rn=(size_t)(E[i].b-E[i].a), cn=(size_t)(o->b-o->a);+ uint8_t *dec=xmalloc(rn); size_t rl=0;+ if(cfse_decompress((uint8_t*)ob+ods+o->a,cn,dec,rn,&rl) || rl!=rn ||+ memcmp(dec,in+ds+E[i].a,rn)){+ fprintf(stderr,"VERIFY FAIL: %s non identico dopo il round-trip\n",E[i].name); return 1; }+ free(dec); rawtot+=(int64_t)rn; comptot+=(int64_t)cn;+ }+ printf("VERIFY OK: %d tensori BIT-IDENTICI | %lld -> %lld byte (%.3fx)\n",+ NE,(long long)rawtot,(long long)comptot,(double)rawtot/(double)comptot);+ return 0;+ }++ /* --- conversione: comprimi ogni tensore, ricostruisci header con nuovi offset --- */+ uint8_t **blob=xmalloc(sizeof(void*)*(size_t)NE); size_t *bn=xmalloc(sizeof(size_t)*(size_t)NE);+ int64_t rawtot=0, comptot=0;+ for(int i=0;i<NE;i++){+ size_t rn=(size_t)(E[i].b-E[i].a);+ blob[i]=xmalloc(cfse_bound(rn));+ bn[i]=cfse_compress((uint8_t*)in+ds+E[i].a,rn,blob[i],cfse_bound(rn));+ if(!bn[i]){ fprintf(stderr,"%s: compress fallita su %s\n",inp,E[i].name); return 1; }+ /* paranoia in linea: round-trip IMMEDIATO prima di scrivere qualsiasi cosa */+ uint8_t *chk=xmalloc(rn); size_t rl=0;+ if(cfse_decompress(blob[i],bn[i],chk,rn,&rl)||rl!=rn||memcmp(chk,in+ds+E[i].a,rn)){+ fprintf(stderr,"ABORT: self-check fallito su %s — nessun file scritto\n",E[i].name); return 1; }+ free(chk); rawtot+=(int64_t)rn; comptot+=(int64_t)bn[i];+ }++ /* header JSON: __metadata__.cfse=1 + tensori con offset compressi (ordine originale) */+ FILE *o=fopen(outp,"wb"); if(!o){perror(outp);return 1;}+ fseek(o,8,SEEK_SET); /* hlen scritto alla fine */+ long h0=ftell(o);+ fprintf(o,"{\"__metadata__\":{\"cfse\":\"1\"}");+ int64_t cur=0;+ for(int i=0;i<NE;i++){+ fprintf(o,",\"%s\":{\"dtype\":\"%s\",\"shape\":",E[i].name,E[i].dtype);+ emit_json_shape(o,E[i].shape);+ fprintf(o,",\"data_offsets\":[%lld,%lld]}",(long long)cur,(long long)(cur+(int64_t)bn[i]));+ cur+=(int64_t)bn[i];+ }+ fputc('}',o);+ long h1=ftell(o);+ while((h1-h0)%8){ fputc(' ',o); h1++; } /* padding a 8 per buona educazione */+ uint64_t hlen=(uint64_t)(h1-h0);+ for(int i=0;i<NE;i++) if(fwrite(blob[i],1,bn[i],o)!=bn[i]){perror("fwrite");return 1;}+ fseek(o,0,SEEK_SET); fwrite(&hlen,8,1,o);+ fclose(o);+ printf("%s: %d tensori | %lld -> %lld byte (%.3fx)\n",+ outp,NE,(long long)rawtot,(long long)comptot,(double)rawtot/(double)comptot);+ return 0;+}diff --git a/c/fse_coli.h b/c/fse_coli.hnew file mode 100644index 00000000..3935fe44--- /dev/null+++ b/c/fse_coli.h@@ -0,0 +1,173 @@+/* fse_coli.h — entropy coder di casa per il container colibrì (LOCALE, non pushato).+ *+ * COSA: rANS statico ordine-0 sui NIBBLE (16 simboli), 2 stati interleaved,+ * frequenze normalizzate a 4096 (12 bit), rinormalizzazione a byte.+ * PERCHE' proprio questo: i pesi int4 sono statisticamente BIANCHI (misurato+ * 2026-07-17: condizionali +0.000, MI tra tensori 0.0009-0.0018 bit), quindi+ * l'ordine-0 e' GIA' ottimo — H=2.924 bit/peso, nessun modello di contesto puo'+ * fare meglio, e' un teorema, non una scelta. Ratio atteso ~1.37x, verificato+ * con zstd (stadio FSE) a 1.371x medio su expert reali.+ *+ * SICUREZZA (questo codice tocca i PESI: un bug qui corrompe l'output del+ * modello in silenzio):+ * - il decoder NON legge mai oltre il buffer: ogni fetch e' bounds-checked;+ * - lo stato finale dei due rANS DEVE tornare a RANS_L: sigillo d'integrita'+ * che intercetta troncamenti e corruzioni (non e' un CRC, ma un flip nel+ * payload disallinea lo stato con probabilita' ~1-2^-46);+ * - la tabella delle frequenze deve sommare ESATTAMENTE a 4096 o si rifiuta;+ * - input incomprimibile -> modalita' raw (mai espandere oltre header);+ * - niente malloc: lavora nei buffer del chiamante.+ *+ * Formato: "CFS1" | mode u8 (0=raw 1=rans) | rawlen u32LE |+ * mode1: freq[16] u16LE | xA u32LE | xB u32LE | payload+ * (l'encoder scrive il payload ALL'INDIETRO: il decoder legge in avanti)+ */+#ifndef FSE_COLI_H+#define FSE_COLI_H+#include <stdint.h>+#include <stddef.h>+#include <string.h>++#define CFSE_PROB_BITS 12+#define CFSE_PROB_SCALE (1u<<CFSE_PROB_BITS)+#define CFSE_RANS_L (1u<<23)+#define CFSE_HDR 9 /* magic4 + mode1 + rawlen4 */++static inline size_t cfse_bound(size_t n){ return n + CFSE_HDR + 64; }++/* conteggi -> frequenze che sommano ESATTAMENTE a 4096; ogni simbolo presente+ * riceve almeno 1 (un simbolo con freq 0 presente nei dati = encode impossibile). */+static int cfse_normalize(const uint64_t cnt[16], uint16_t freq[16]){+ uint64_t tot=0; int used=0;+ for(int i=0;i<16;i++){ tot+=cnt[i]; if(cnt[i]) used++; }+ if(!tot) return -1;+ uint32_t sum=0; int last=-1;+ for(int i=0;i<16;i++){+ if(!cnt[i]){ freq[i]=0; continue; }+ uint64_t f=(cnt[i]*CFSE_PROB_SCALE)/tot; if(!f) f=1;+ if(f>CFSE_PROB_SCALE-(unsigned)(used-1)) f=CFSE_PROB_SCALE-(used-1);+ freq[i]=(uint16_t)f; sum+=(uint32_t)f; last=i;+ }+ /* aggiusta sul simbolo piu' frequente (mai sotto 1) */+ int big=last; for(int i=0;i<16;i++) if(freq[i]>freq[big]) big=i;+ int32_t diff=(int32_t)CFSE_PROB_SCALE-(int32_t)sum;+ if((int32_t)freq[big]+diff<1) return -1;+ freq[big]=(uint16_t)((int32_t)freq[big]+diff);+ return 0;+}++/* comprime n byte (2n nibble). Ritorna i byte scritti, 0 = errore/cap corto. */+static size_t cfse_compress(const uint8_t *in, size_t n, uint8_t *out, size_t cap){+ if(cap<cfse_bound(n) || n>0xFFFFFFFFu) return 0;+ memcpy(out,"CFS1",4); out[5]=(uint8_t)(n); out[6]=(uint8_t)(n>>8);+ out[7]=(uint8_t)(n>>16); out[8]=(uint8_t)(n>>24);+ if(n==0){ out[4]=0; return CFSE_HDR; }++ uint64_t cnt[16]={0};+ for(size_t i=0;i<n;i++){ cnt[in[i]&0xF]++; cnt[in[i]>>4]++; }+ uint16_t freq[16]; uint32_t cum[17]={0};+ if(cfse_normalize(cnt,freq)) goto raw;+ for(int i=0;i<16;i++) cum[i+1]=cum[i]+freq[i];++ {+ /* encode all'indietro nel fondo di out; poi compattiamo dietro l'header */+ uint8_t *base=out+CFSE_HDR+32; /* header + tabella freq */+ uint8_t *end=out+cap, *p=end;+ uint32_t xA=CFSE_RANS_L, xB=CFSE_RANS_L;+ size_t N=2*n; /* nibble totali */+ for(size_t i=N; i-- > 0; ){ /* i = N-1 .. 0 */+ unsigned s = (i&1) ? (in[i>>1]>>4) : (in[i>>1]&0xF);+ uint32_t *x = (i&1) ? &xB : &xA; /* pari->A, dispari->B (specchio del decode) */+ uint32_t f=freq[s];+ uint32_t xmax=((CFSE_RANS_L>>CFSE_PROB_BITS)<<8)*f;+ while(*x>=xmax){ if(p<=base) goto raw; *--p=(uint8_t)(*x&0xFF); *x>>=8; }+ *x = ((*x/f)<<CFSE_PROB_BITS) + (*x%f) + cum[s];+ }+ /* flush: B poi A scrivendo all'indietro -> nel flusso in avanti arriva A poi B */+ if(p-base<8) goto raw;+ for(int k=3;k>=0;k--) *--p=(uint8_t)(xB>>(8*k));+ for(int k=3;k>=0;k--) *--p=(uint8_t)(xA>>(8*k));+ size_t payload=(size_t)(end-p);+ if(CFSE_HDR+32+payload >= n+CFSE_HDR) goto raw; /* non conviene */+ out[4]=1;+ for(int i=0;i<16;i++){ out[CFSE_HDR+2*i]=(uint8_t)freq[i]; out[CFSE_HDR+2*i+1]=(uint8_t)(freq[i]>>8); }+ memmove(out+CFSE_HDR+32, p, payload);+ return CFSE_HDR+32+payload;+ }+raw:+ out[4]=0; memcpy(out+CFSE_HDR,in,n); return CFSE_HDR+n;+}++/* 0 = ok (rawlen scritto), -1 = input invalido/corrotto/troncato. MAI legge+ * oltre in+nin ne' scrive oltre out+cap. */+static int cfse_decompress(const uint8_t *in, size_t nin, uint8_t *out, size_t cap, size_t *rawlen){+ if(nin<CFSE_HDR || memcmp(in,"CFS1",4)) return -1;+ size_t n = (size_t)in[5] | ((size_t)in[6]<<8) | ((size_t)in[7]<<16) | ((size_t)in[8]<<24);+ if(n>cap) return -1;+ *rawlen=n;+ if(in[4]==0){+ if(nin != CFSE_HDR+n) return -1;+ memcpy(out,in+CFSE_HDR,n); return 0;+ }+ if(in[4]!=1 || n==0 || nin<CFSE_HDR+32+8) return -1;++ uint16_t freq[16]; uint32_t cum[17]={0}; uint32_t sum=0;+ for(int i=0;i<16;i++){ freq[i]=(uint16_t)(in[CFSE_HDR+2*i] | (in[CFSE_HDR+2*i+1]<<8)); sum+=freq[i]; }+ if(sum!=CFSE_PROB_SCALE) return -1;+ for(int i=0;i<16;i++) cum[i+1]=cum[i]+freq[i];+ /* tabella unica per slot: bias(12b)<<20 | freq(13b)<<7 | sym(4b) = 29 bit.+ * ATTENZIONE al layout: freq puo' valere 4096 (caso un-solo-simbolo) e vuole+ * 13 bit — messa a <<20 overflowava il uint32 e la batteria l'ha beccato.+ * bias = slot - cum[sym]: il passo di decode diventa UNA lookup L1 (16 KB)+ * invece di tre dipendenti (slot2sym -> freq[s] -> cum[s]). */+ uint32_t tab[CFSE_PROB_SCALE];+ for(int s=0;s<16;s++)+ for(uint32_t slot=cum[s]; slot<cum[s+1]; slot++)+ tab[slot] = ((slot-cum[s])<<20) | ((uint32_t)freq[s]<<7) | (uint32_t)s;++ const uint8_t *p=in+CFSE_HDR+32, *end=in+nin;+ /* gli stati sono nel flusso in LITTLE-endian (l'encoder li scrive+ * all'indietro byte alto per primo -> in avanti escono LE) */+ uint32_t xA=(uint32_t)p[0]|((uint32_t)p[1]<<8)|((uint32_t)p[2]<<16)|((uint32_t)p[3]<<24);+ uint32_t xB=(uint32_t)p[4]|((uint32_t)p[5]<<8)|((uint32_t)p[6]<<16)|((uint32_t)p[7]<<24);+ p+=8;+ if(xA<CFSE_RANS_L || xB<CFSE_RANS_L) return -1;++ /* Il ciclo caldo lavora a COPPIE (nibble pari->xA, dispari->xB) in registri.+ * Ogni coppia consuma AL MASSIMO 4 byte di payload (2 per stato: da x>=2^11+ * dopo il passo, due rinormalizzazioni a byte riportano sopra 2^23).+ * Quindi: finche' restano >=4*coppie byte, si corre SENZA bounds-check per+ * byte (la matematica lo garantisce, non la fortuna); la coda torna al+ * percorso controllato byte-per-byte. Sicurezza invariata: stesso formato,+ * stessi sigilli (p==end, stati==RANS_L), la batteria + ASAN lo certificano. */+ uint32_t xa=xA, xb=xB;+ size_t pairs=n, j=0;+ while(j<pairs){+ size_t safe = (size_t)(end-p)/4; /* coppie garantite senza check */+ size_t stop = j + (pairs-j < safe ? pairs-j : safe);+ for(; j<stop; j++){+ uint32_t ea = tab[xa & (CFSE_PROB_SCALE-1)];+ xa = ((ea>>7)&0x1FFF)*(xa>>CFSE_PROB_BITS) + (ea>>20);+ if(xa<CFSE_RANS_L){ xa=(xa<<8)|*p++; if(xa<CFSE_RANS_L) xa=(xa<<8)|*p++; }+ uint32_t eb = tab[xb & (CFSE_PROB_SCALE-1)];+ xb = ((eb>>7)&0x1FFF)*(xb>>CFSE_PROB_BITS) + (eb>>20);+ if(xb<CFSE_RANS_L){ xb=(xb<<8)|*p++; if(xb<CFSE_RANS_L) xb=(xb<<8)|*p++; }+ out[j] = (uint8_t)((ea&0xF) | ((eb&0xF)<<4));+ }+ if(j<pairs){ /* coda: percorso CONTROLLATO */+ uint32_t ea = tab[xa & (CFSE_PROB_SCALE-1)];+ xa = ((ea>>7)&0x1FFF)*(xa>>CFSE_PROB_BITS) + (ea>>20);+ while(xa<CFSE_RANS_L){ if(p>=end) return -1; xa=(xa<<8)|*p++; }+ uint32_t eb = tab[xb & (CFSE_PROB_SCALE-1)];+ xb = ((eb>>7)&0x1FFF)*(xb>>CFSE_PROB_BITS) + (eb>>20);+ while(xb<CFSE_RANS_L){ if(p>=end) return -1; xb=(xb<<8)|*p++; }+ out[j] = (uint8_t)((ea&0xF) | ((eb&0xF)<<4));+ j++;+ }+ }+ xA=xa; xB=xb; /* per i sigilli finali */+ if(p!=end) return -1; /* byte avanzati = flusso non nostro */+ if(xA!=CFSE_RANS_L || xB!=CFSE_RANS_L) return -1; /* sigillo d'integrita' */+ return 0;+}+#endifdiff --git a/c/tests/test_fse.c b/c/tests/test_fse.cnew file mode 100644index 00000000..dcf862e0--- /dev/null+++ b/c/tests/test_fse.c@@ -0,0 +1,128 @@+/* Batteria di test per fse_coli.h — scritta PRIMA che il codec tocchi glm.c.+ *+ * Questo codec decomprimera' PESI: un bug qui non crasha, corrompe l'output+ * del modello in silenzio. Quindi: round-trip esatti, distribuzioni degeneri,+ * fuzz di troncamento e corruzione (il decoder non deve MAI crashare, leggere+ * oltre, o accettare in silenzio un flusso rotto), expert reale se disponibile+ * (env EXPERT_RAW), ratio contro l'entropia misurata, velocita' di decode.+ * Da compilare anche con -fsanitize=address,undefined: il fuzz sotto ASAN e'+ * il vero certificato di bounds-safety. */+#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <time.h>+#include "../fse_coli.h"++static uint64_t rng=0x243F6A8885A308D3ull;+static uint32_t xr(void){ rng^=rng<<13; rng^=rng>>7; rng^=rng<<17; return (uint32_t)(rng>>32); }++static int roundtrip(const uint8_t *in, size_t n, const char *name, double *ratio){+ size_t cap=cfse_bound(n);+ uint8_t *c=malloc(cap?cap:1), *d=malloc(n?n:1);+ size_t cs=cfse_compress(in,n,c,cap);+ if(!cs){ printf(" FAIL %s: compress=0\n",name); return 1; }+ size_t rl=0;+ if(cfse_decompress(c,cs,d,n,&rl)){ printf(" FAIL %s: decompress rifiutato\n",name); return 1; }+ if(rl!=n || (n && memcmp(in,d,n))){ printf(" FAIL %s: round-trip NON identico\n",name); return 1; }+ if(ratio) *ratio = cs? (double)n/(double)cs : 0;+ free(c); free(d); return 0;+}++int main(void){+ int fail=0; double r;+ printf("test_fse: rANS nibble per il container colibri'\n");++ /* --- 1. vettori noti e casi limite --- */+ { uint8_t v[]={0x00,0x11,0x22,0x33,0x44,0x55,0x66,0x77,0x88,0x99,0xAA,0xBB,0xCC,0xDD,0xEE,0xFF};+ fail|=roundtrip(v,sizeof v,"16 byte tutti i nibble",NULL); }+ fail|=roundtrip((const uint8_t*)"",0,"vuoto",NULL);+ { uint8_t b=0x5A; fail|=roundtrip(&b,1,"un byte",NULL); }+ { uint8_t v[3]={0xFF,0x00,0xFF}; fail|=roundtrip(v,3,"tre byte estremi",NULL); }+ if(!fail) printf(" vettori noti + vuoto + 1 byte ok\n");++ /* --- 2. degeneri: un solo simbolo, due simboli --- */+ { uint8_t *v=malloc(100000); memset(v,0x88,100000);+ fail|=roundtrip(v,100000,"tutto un simbolo",&r);+ if(!fail) printf(" degenere un-simbolo: ratio %.0fx ok\n",r);+ for(size_t i=0;i<100000;i++) v[i]=(xr()&1)?0x87:0x78;+ fail|=roundtrip(v,100000,"due simboli",&r);+ if(!fail) printf(" degenere due-simboli: ratio %.2fx ok\n",r);+ free(v); }++ /* --- 3. uniforme (incomprimibile): DEVE cadere in raw, mai espandere --- */+ { size_t n=1<<20; uint8_t *v=malloc(n);+ for(size_t i=0;i<n;i++) v[i]=(uint8_t)xr();+ size_t cap=cfse_bound(n); uint8_t *c=malloc(cap);+ size_t cs=cfse_compress(v,n,c,cap);+ if(!cs || cs>n+CFSE_HDR){ printf(" FAIL uniforme: espande (%zu > %zu)\n",cs,n+CFSE_HDR); fail=1; }+ fail|=roundtrip(v,n,"uniforme 1MB",&r);+ if(!fail) printf(" uniforme 1MB: ratio %.3fx (fallback raw) ok\n",r);+ free(v); free(c); }++ /* --- 4. gaussiana tipo-pesi: il ratio deve avvicinare il previsto 1.37x --- */+ { size_t n=4<<20; uint8_t *v=malloc(n);+ for(size_t i=0;i<n;i++){+ int a=((int)(xr()&15)+(int)(xr()&15)+(int)(xr()&15)+(int)(xr()&15))/4; /* ~gauss 0..15 */+ int b=((int)(xr()&15)+(int)(xr()&15)+(int)(xr()&15)+(int)(xr()&15))/4;+ v[i]=(uint8_t)(a|(b<<4)); }+ fail|=roundtrip(v,n,"gaussiana 4MB",&r);+ if(!fail) printf(" gaussiana 4MB: ratio %.3fx ok\n",r);+ free(v); }++ /* --- 5. taglie casuali: 400 round-trip di lunghezze 0..8191 --- */+ { int bad=0;+ for(int t=0;t<400;t++){ size_t n=xr()&8191; uint8_t *v=malloc(n?n:1);+ for(size_t i=0;i<n;i++){ int a=((int)(xr()&15)+(int)(xr()&15))/2; v[i]=(uint8_t)(a|(((int)(xr()&7)+4)<<4)); }+ bad|=roundtrip(v,n,"taglia casuale",NULL); free(v); }+ fail|=bad; if(!bad) printf(" 400 round-trip a taglie casuali ok\n"); }++ /* --- 6. FUZZ troncamento: OGNI prefisso del compresso va rifiutato pulito --- */+ { size_t n=65536; uint8_t *v=malloc(n);+ for(size_t i=0;i<n;i++){ int a=((int)(xr()&15)+(int)(xr()&15))/2; v[i]=(uint8_t)(a|(a<<4)); }+ size_t cap=cfse_bound(n); uint8_t *c=malloc(cap), *d=malloc(n);+ size_t cs=cfse_compress(v,n,c,cap); size_t rl;+ int accepted=0;+ for(size_t L=0;L<cs;L++) /* TUTTI i troncamenti */+ if(cfse_decompress(c,L,d,n,&rl)==0) accepted++;+ if(accepted){ printf(" FAIL troncamento: %d prefissi accettati\n",accepted); fail=1; }+ else printf(" troncamento: %zu prefissi, tutti rifiutati ok\n",cs);++ /* --- 7. FUZZ corruzione: flip di byte; mai crash, (quasi) mai accettato --- */+ int acc=0, trials=2000;+ for(int t=0;t<trials;t++){+ size_t pos=xr()%cs; uint8_t old=c[pos];+ c[pos]^=(uint8_t)(1+(xr()&0xFE));+ if(cfse_decompress(c,cs,d,n,&rl)==0 && (rl!=n || memcmp(v,d,n))) acc++;+ c[pos]=old; }+ printf(" corruzione: %d flip, %d accettati con dati sbagliati %s\n",+ trials,acc, acc<=2?"ok":"TROPPI");+ if(acc>2) fail=1; /* sigillo: attesi ~0 */+ free(v); free(c); free(d); }++ /* --- 8. expert REALE (se disponibile) + velocita' --- */+ const char *er=getenv("EXPERT_RAW");+ if(er){+ FILE *f=fopen(er,"rb");+ if(f){ fseek(f,0,SEEK_END); size_t n=(size_t)ftell(f); fseek(f,0,SEEK_SET);+ uint8_t *v=malloc(n); if(fread(v,1,n,f)!=n){printf(" FAIL lettura expert\n");return 1;} fclose(f);+ size_t cap=cfse_bound(n); uint8_t *c=malloc(cap), *d=malloc(n);+ size_t cs=cfse_compress(v,n,c,cap); size_t rl;+ if(!cs||cfse_decompress(c,cs,d,n,&rl)||rl!=n||memcmp(v,d,n)){+ printf(" FAIL expert reale: round-trip\n"); fail=1;+ } else {+ struct timespec t0,t1; double best=1e9;+ for(int k=0;k<5;k++){ clock_gettime(CLOCK_MONOTONIC,&t0);+ cfse_decompress(c,cs,d,n,&rl);+ clock_gettime(CLOCK_MONOTONIC,&t1);+ double dt=(t1.tv_sec-t0.tv_sec)+(t1.tv_nsec-t0.tv_nsec)/1e9;+ if(dt<best) best=dt; }+ printf(" EXPERT REALE (%zu byte): ratio %.3fx | decode %.0f ms = %.2f GB/s (1 core)\n",+ n,(double)n/cs,best*1000,n/best/1e9);+ }+ free(v);free(c);free(d);+ } else printf(" (EXPERT_RAW non leggibile: salto)\n");+ } else printf(" (EXPERT_RAW non impostato: salto il test su pesi reali)\n");++ printf(fail? "test_fse: FAIL\n" : "test_fse: ok\n");+ return fail;+}