Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(kv): retain committed prefix after save failures

Rudy Celeklie50064e8ed33
diff --git a/c/kv_persist.h b/c/kv_persist.h
index 2ad5ffb7..a965579f 100644
--- a/c/kv_persist.h
+++ b/c/kv_persist.h
@@ -28,20 +28,41 @@ static int64_t kv_rec_bytes(Model *m){
return rec;
}
+/* Publishing nrec is the commit step: neither a buffered write nor fflush
+ * alone proves the corresponding records reached the persistence layer. */
+static int kv_disk_sync(FILE *f){
+ if(fflush(f)!=0) return 0;
+#ifdef _WIN32
+ return _commit(_fileno(f))==0;
+#else
+ return fsync(fileno(f))==0;
+#endif
+}
+
+static void kv_disk_write_failed(KVState *k){
+ fprintf(stderr,"[KV] save failed: %s (will retry on the next save)\n",strerror(errno));
+ if(k->disk_fp){ fclose(k->disk_fp); k->disk_fp=NULL; }
+}
+
static int kv_disk_open(Model *m){
KVState *k=m->kv;
if(k->disk_fp) return 1;
k->disk_fp=fopen(k->disk_path,"r+b");
- if(k->disk_fp){ char mg[8]; /* formato del file != formato attivo -> riscrivi */
- if(fread(mg,1,8,k->disk_fp)!=8 || memcmp(mg,kv_active_magic(),8)){
+ if(k->disk_fp){ char mg[8]; int32_t h[8], want[8]; kv_hdr(m,want,0);
+ /* A failed initial header write may leave the magic but no valid
+ * dimensions. Do not append rows to that incomplete file. */
+ if(fread(mg,1,8,k->disk_fp)!=8 || memcmp(mg,kv_active_magic(),8) ||
+ fread(h,4,8,k->disk_fp)!=8 || memcmp(h,want,6*sizeof(int32_t)) || h[7]!=want[7]){
fclose(k->disk_fp); k->disk_fp=NULL; k->disk_nrec=0; }
}
if(!k->disk_fp){
k->disk_fp=fopen(k->disk_path,"wb");
if(!k->disk_fp) return 0;
+ k->disk_nrec=0; /* a newly created file contains no committed rows */
int32_t h[8]; kv_hdr(m,h,0);
- fwrite(kv_active_magic(),1,8,k->disk_fp); fwrite(h,4,8,k->disk_fp);
- fflush(k->disk_fp);
+ if(fwrite(kv_active_magic(),1,8,k->disk_fp)!=8 ||
+ fwrite(h,4,8,k->disk_fp)!=8 || !kv_disk_sync(k->disk_fp)){
+ kv_disk_write_failed(k); return 0; }
fclose(k->disk_fp);
k->disk_fp=fopen(k->disk_path,"r+b");
if(!k->disk_fp) return 0;
@@ -60,9 +81,16 @@ static void kv_disk_truncate(Model *m, int nrec){
if(k->disk_fp){ fclose(k->disk_fp); k->disk_fp=NULL; }
FILE *f=fopen(k->disk_path,"r+b");
if(!f){ k->disk_nrec=0; return; }
- k->disk_nrec=nrec;
- int32_t nr=nrec; fseek(f,8+6*4,SEEK_SET); fwrite(&nr,4,1,f);
- fflush(f); fclose(f);
+ int32_t nr=nrec;
+ int ok=fseek(f,8+6*4,SEEK_SET)==0 && fwrite(&nr,4,1,f)==1 && kv_disk_sync(f);
+ if(fclose(f)!=0) ok=0;
+ if(ok) k->disk_nrec=nrec;
+ else {
+ /* The desired prefix is shorter than the saved one: retry every live
+ * row next time, rather than letting the old count suppress a save. */
+ k->disk_nrec=0;
+ fprintf(stderr,"[KV] could not persist the conversation's shorter prefix\n");
+ }
}
static void kv_disk_reset(Model *m){ kv_disk_truncate(m,0); }
@@ -79,7 +107,7 @@ static void kv_disk_append(Model *m, const int *hist, int len){
if(!nb) return;
k->disk_buf=nb; k->disk_buf_cap=rec;
}
- fseek(f, 8+8*4 + (int64_t)k->disk_nrec*rec, SEEK_SET);
+ if(fseek(f, 8+8*4 + (int64_t)k->disk_nrec*rec, SEEK_SET)!=0) goto write_failed;
for(int p=k->disk_nrec;p<len;p++){
uint8_t *b=k->disk_buf;
*(int32_t*)b = hist[p]; b+=4;
@@ -103,23 +131,18 @@ static void kv_disk_append(Model *m, const int *hist, int len){
if(m->has_dsa) for(int i=0;i<c->n_layers;i++) if(m->Ic[i]){
memcpy(b, m->Ic[i]+(int64_t)p*c->index_hd, (size_t)c->index_hd*4); b+=c->index_hd*4;
}
- fwrite(k->disk_buf, 1, (size_t)rec, f);
+ if(fwrite(k->disk_buf, 1, (size_t)rec, f)!=(size_t)rec) goto write_failed;
}
- fflush(f); /* dati prima, contatore poi */
-#ifdef _WIN32
- _commit(_fileno(f));
-#else
- fsync(fileno(f)); /* i DATI su disco prima che il contatore li
- * dichiari: fflush ferma solo alla page cache */
-#endif
- int32_t nr=len; fseek(f,8+6*4,SEEK_SET); fwrite(&nr,4,1,f);
- fflush(f); /* persist the counter too */
-#ifdef _WIN32
- _commit(_fileno(f));
-#else
- fsync(fileno(f));
-#endif
+ if(!kv_disk_sync(f)) goto write_failed; /* data before the counter */
+ int32_t nr=len;
+ if(fseek(f,8+6*4,SEEK_SET)!=0 || fwrite(&nr,4,1,f)!=1 || !kv_disk_sync(f))
+ goto write_failed;
k->disk_nrec=len;
+ return;
+write_failed:
+ /* Keep the committed count. Closing clears the failed stdio stream, so a
+ * later save rewrites the incomplete tail instead of skipping it. */
+ kv_disk_write_failed(k);
}
/* Bonifica una riga fp8 letta da disco: l'encoder non emette MAI i codici NaN e4m3
* (0x7F/0xFF), ma un file corrotto potrebbe — la GPU (__nv_cvt) li decodifica NaN. */
diff --git a/c/tests/test_kv_disk.c b/c/tests/test_kv_disk.c
index c6c7b4a0..b2efd6c2 100644
--- a/c/tests/test_kv_disk.c
+++ b/c/tests/test_kv_disk.c
@@ -37,6 +37,53 @@ int main(void){
for(int j=0;j<m.c.kv_lora;j++)
CHECK(m.Lc[i][(int64_t)p*m.c.kv_lora+j]==fill(i,p,j), "v1 Lc[%d] p%d j%d", i,p,j);
+#ifndef _WIN32
+ /* A real failed data flush must not publish an unwritten tail or suppress
+ * the next save. Keep two committed rows, then deny extending the file. */
+ if(m.kv->disk_fp){ fclose(m.kv->disk_fp); m.kv->disk_fp=NULL; }
+ kv_disk_truncate(&m,2);
+ int64_t committed_bytes=40+2*kv_rec_bytes(&m);
+ CHECK(truncate(PATH,(off_t)committed_bytes)==0,"truncate test fixture");
+ struct rlimit previous, limited;
+ CHECK(getrlimit(RLIMIT_FSIZE,&previous)==0,"read file-size limit");
+ limited=previous; limited.rlim_cur=(rlim_t)committed_bytes;
+ void (*previous_signal)(int)=signal(SIGXFSZ,SIG_IGN);
+ CHECK(setrlimit(RLIMIT_FSIZE,&limited)==0,"set file-size limit");
+ kv_disk_append(&m,hist,NP);
+ CHECK(setrlimit(RLIMIT_FSIZE,&previous)==0,"restore file-size limit");
+ signal(SIGXFSZ,previous_signal);
+ CHECK(m.kv->disk_nrec==2,"failed append must retain committed nrec=2, got %d",m.kv->disk_nrec);
+ { FILE *f=fopen(PATH,"rb"); int32_t nr=-1;
+ CHECK(f && fseek(f,32,SEEK_SET)==0 && fread(&nr,4,1,f)==1 && nr==2,
+ "failed data flush must leave header nrec=2, got %d",nr);
+ if(f) fclose(f); }
+ kv_disk_append(&m,hist,NP);
+ CHECK(m.kv->disk_nrec==NP,"retry must persist the unwritten rows");
+ kv_alloc(&m,16); m.kv->disk_nrec=0;
+ CHECK(kv_disk_load(&m,hist2,16)==NP,"retry load must include all rows");
+ for(int p=0;p<NP;p++) CHECK(hist2[p]==hist[p],"retry hist[%d]",p);
+
+ /* Failure while creating the first header must also recover; its magic
+ * alone does not make the truncated file a valid persistence target. */
+ if(m.kv->disk_fp){ fclose(m.kv->disk_fp); m.kv->disk_fp=NULL; }
+ snprintf(m.kv->disk_path,sizeof(m.kv->disk_path),"%s.init",PATH);
+ remove(m.kv->disk_path); m.kv->disk_nrec=0;
+ limited.rlim_cur=30;
+ previous_signal=signal(SIGXFSZ,SIG_IGN);
+ CHECK(setrlimit(RLIMIT_FSIZE,&limited)==0,"limit initial header write");
+ kv_disk_append(&m,hist,NP);
+ CHECK(setrlimit(RLIMIT_FSIZE,&previous)==0,"restore initial header limit");
+ signal(SIGXFSZ,previous_signal);
+ CHECK(m.kv->disk_nrec==0,"failed header cannot commit rows");
+ kv_disk_append(&m,hist,NP);
+ kv_alloc(&m,16); m.kv->disk_nrec=0;
+ CHECK(kv_disk_load(&m,hist2,16)==NP,"partial header must be recreated before retry");
+ for(int p=0;p<NP;p++) CHECK(hist2[p]==hist[p],"header retry hist[%d]",p);
+ if(m.kv->disk_fp){ fclose(m.kv->disk_fp); m.kv->disk_fp=NULL; }
+ remove(m.kv->disk_path);
+ snprintf(m.kv->disk_path,sizeof(m.kv->disk_path),"%s",PATH);
+#endif
+
/* ---- v1 file + KV8: quantize-on-load; il file v1 resta INTATTO (un crash
* prima del primo save non deve perdere la conversazione) ---- */
if(m.kv->disk_fp){ fclose(m.kv->disk_fp); m.kv->disk_fp=NULL; } /* "riavvio" del processo */