Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

stamp: conflicting duplicate claims refuse at ingest (was first-wins)

monotophic38c2185fd3e5
diff --git a/c/st.h b/c/st.h
index 2a22df54..d4d47616 100644
--- a/c/st.h
+++ b/c/st.h
@@ -300,6 +300,34 @@ static void st_fmt_stamp_ingest(shards *S, jval *root, const char *shard_path) {
if (v->t != J_STR) {
fprintf(stderr, "%s: colibri.fmt entry '%s' is not a string -- malformed stamp, refusing (untrusted container)\n",
shard_path, inner->keys[i]); exit(1); }
+ /* DUPLICATE CLAIMS (user-ratified design, register D8): at most one
+ * DISTINCT format claim per tensor name, container-wide. An entry
+ * that repeats an already-ingested claim verbatim is tolerated and
+ * collapsed to one entry (idempotent -- a centralized-manifest
+ * writer may legally stamp the same map into every shard, and a
+ * shard may stamp tensors it does not itself contain; there is no
+ * locality constraint). An entry that CONTRADICTS an earlier claim
+ * refuses by name: a container that disagrees with itself about a
+ * tensor's format is corrupted or hostile, and the previous
+ * first-wins behavior made the outcome depend on shard enumeration
+ * order (st_scan_dir is raw readdir order, not sorted) while
+ * mis-diagnosing the real problem downstream as a stamp/inference
+ * mismatch -- or hiding it entirely when the enumeration happened
+ * to favor the agreeing claim. The refusal names the tensor and
+ * both format names; the EARLIER claim's shard file is not named
+ * (per-entry shard provenance isn't stored, and adding it just for
+ * this message would be new plumbing -- only the current shard's
+ * path is in scope here). Linear rescan per entry is O(n^2) worst
+ * case, bounded by ST_FMT_STAMP_MAX at one-time startup. */
+ int dup = -1;
+ for (int k = 0; k < S->fmt_n; k++)
+ if (!strcmp(S->fmt_name[k], inner->keys[i])) { dup = k; break; }
+ if (dup >= 0) {
+ if (!strcmp(S->fmt_val[dup], v->str)) continue; /* agreeing duplicate: keep one */
+ fprintf(stderr, "%s: __metadata__[\"colibri.fmt\"] stamps tensor '%s' as '%s', but an "
+ "earlier shard's map already stamped it '%s' -- conflicting format claims, "
+ "refusing (untrusted container)\n",
+ shard_path, inner->keys[i], v->str, S->fmt_val[dup]); exit(1); }
if (S->fmt_n >= ST_FMT_STAMP_MAX) {
fprintf(stderr, "%s: __metadata__[\"colibri.fmt\"] stamps more than %d tensor names across "
"this container's shards -- stamps are a resident-tensor convention (docs/FORMATS.md), "
diff --git a/c/tests/test_fp8_load.c b/c/tests/test_fp8_load.c
index 4a509d26..6fcd9361 100644
--- a/c/tests/test_fp8_load.c
+++ b/c/tests/test_fp8_load.c
@@ -997,6 +997,87 @@ static void test_stamp_absent(void){
char p[300]; snprintf(p,sizeof p,"%s/model.safetensors",dir); unlink(p); rmdir(dir);
}
+/* ---- Duplicate stamp claims (micro-round 2, user-ratified design D8) ----
+ * At most one DISTINCT format claim per tensor name, container-wide:
+ * conflicting claims refuse at ingest (naming the tensor and both format
+ * names); agreeing duplicates are tolerated (idempotent). There is no
+ * locality constraint -- shard 2 below stamps "w", a tensor it does NOT
+ * contain, which is legal by design (centralized-manifest writers) and is
+ * exactly what makes cross-shard conflicts possible in the first place. */
+
+/* Shard 1 = write_stamp_fixture's single-tensor fp8 shard ("w" + "w.qs"),
+ * renamed into a two-shard layout; shard 2 = one tiny aux f32 tensor of its
+ * own plus a colibri.fmt map stamping "w". */
+static void write_second_stamp_shard(const char *dir, const char *meta_json_string_literal){
+ char path[300]; snprintf(path,sizeof path,"%s/model-extra.safetensors",dir);
+ static float aux[4] = {1.f,2.f,3.f,4.f};
+ char hdr[1024]; int hl;
+ hl=snprintf(hdr,sizeof hdr,
+ "{\"__metadata__\":{\"colibri.fmt\":%s},"
+ "\"aux\":{\"dtype\":\"F32\",\"shape\":[4],\"data_offsets\":[0,16]}}",
+ meta_json_string_literal);
+ FILE *f=fopen(path,"wb");
+ if(!f){ printf("FAIL: cannot create %s\n", path); fails++; return; }
+ uint64_t hlen=(uint64_t)hl;
+ fwrite(&hlen,8,1,f); fwrite(hdr,1,hl,f); fwrite(aux,1,sizeof aux,f);
+ fclose(f);
+}
+static void rm_two_shard_fixture(const char *dir){
+ char p[300];
+ snprintf(p,sizeof p,"%s/model.safetensors",dir); unlink(p);
+ snprintf(p,sizeof p,"%s/model-extra.safetensors",dir); unlink(p);
+ rmdir(dir);
+}
+static void test_stamp_conflicting_duplicate(void){
+#ifndef _WIN32
+ const char *dir="tests/tmp_fp8_stamp_conflict";
+ write_stamp_fixture(dir, "\"{\\\"w\\\":\\\"fp8-e4m3-b128\\\"}\"");
+ write_second_stamp_shard(dir, "\"{\\\"w\\\":\\\"int8-row\\\"}\"");
+ /* refusal fires at DISCOVERY time (st_init's header-parse loop), before
+ * any tensor is resolved -- fork st_init alone and capture stderr. */
+ fflush(stdout);
+ int pipefd[2]; if(pipe(pipefd)!=0){ CHECK(0); return; }
+ pid_t pid = fork();
+ if(pid < 0){ CHECK(0); return; }
+ if(pid == 0){
+ dup2(pipefd[1],2); close(pipefd[0]); close(pipefd[1]);
+ static Model gm; memset(&gm,0,sizeof gm);
+ st_init(&gm.S, dir); /* must exit(1) inside the ingest */
+ _exit(42); /* surviving discovery is the bug */
+ }
+ close(pipefd[1]);
+ char err[2048]={0}; size_t eoff=0; ssize_t n;
+ while(eoff<sizeof(err)-1 && (n=read(pipefd[0],err+eoff,sizeof(err)-1-eoff))>0) eoff+=(size_t)n;
+ close(pipefd[0]);
+ int status=0; waitpid(pid,&status,0);
+ int ok = WIFEXITED(status) && WEXITSTATUS(status)==1;
+ if(!ok) printf("FAIL stamp-conflict: expected exit(1) at discovery, got status=%d, stderr=%.300s\n", status, err);
+ CHECK(ok);
+ /* the refusal must NAME the tensor and BOTH claims (either shard may be
+ * enumerated first -- readdir order -- so assert both names, not roles). */
+ CHECK(strstr(err,"conflicting format claims")!=NULL);
+ CHECK(strstr(err,"'w'")!=NULL);
+ CHECK(strstr(err,"fp8-e4m3-b128")!=NULL && strstr(err,"int8-row")!=NULL);
+ CHECK(strstr(err,"refus")!=NULL);
+ rm_two_shard_fixture(dir);
+#else
+ printf("skipped on Windows (no fork): stamp-conflict refusal\n");
+#endif
+}
+static void test_stamp_agreeing_duplicate(void){
+ const char *dir="tests/tmp_fp8_stamp_dupok";
+ write_stamp_fixture(dir, "\"{\\\"w\\\":\\\"fp8-e4m3-b128\\\"}\"");
+ write_second_stamp_shard(dir, "\"{\\\"w\\\":\\\"fp8-e4m3-b128\\\"}\"");
+ static Model gm; memset(&gm,0,sizeof gm);
+ st_init(&gm.S, dir); /* agreeing duplicate: no refusal */
+ CHECK(gm.S.fmt_n == 1); /* collapsed to ONE entry, not two */
+ QT t; memset(&t,0,sizeof t);
+ qt_from_disk(&gm,"w",8,256,8,0,&t);
+ CHECK(t.fmt==8); /* loads exactly as single-stamped */
+ CHECK(t.q8!=NULL && t.s!=NULL);
+ rm_two_shard_fixture(dir);
+}
+
/* ---- Stamp-map scan bound (maintainer review, #529): st_fmt_stamp_ingest
* (st.h) caps the number of stamped-tensor entries it will ingest across a
* container's shards at ST_FMT_STAMP_MAX and refuses (exit(1)) past it --
@@ -1129,6 +1210,8 @@ int main(void){
test_stamp_mismatching();
test_stamp_unrecognized_name();
test_stamp_absent();
+ test_stamp_conflicting_duplicate();
+ test_stamp_agreeing_duplicate();
test_stamp_map_cap_boundary_ok();
test_stamp_map_cap_exceeded();
if(fails){ printf("fp8 loader-seam tests: %d FAILED\n", fails); return 1; }
diff --git a/docs/FORMATS.md b/docs/FORMATS.md
index fbec9f1c..038307fe 100644
--- a/docs/FORMATS.md
+++ b/docs/FORMATS.md
@@ -265,6 +265,29 @@ anything is wrong with a container; it is the default, expected state for
everything that predates this convention (which, as of this PR, is
everything).
+### Duplicate claims, locality, coverage
+
+Three rules complete the stamp's container-wide semantics (user-ratified
+this revision):
+
+- **Conflicting claims refuse.** At most one DISTINCT format claim per
+ tensor name, container-wide. If two shards' `colibri.fmt` maps (or two
+ entries anywhere in the container) stamp the same tensor name with
+ DIFFERENT format names, ingest refuses loudly at discovery time, naming
+ the tensor and both claims β€” a container that disagrees with itself about
+ a tensor's format is corrupted or hostile. (The previous behavior was
+ first-wins, which made the outcome depend on shard enumeration order β€”
+ `st_scan_dir` is raw `readdir` order β€” and mis-diagnosed or hid the real
+ problem.)
+- **Agreeing duplicates are tolerated** (idempotent; collapsed to one
+ entry). A centralized-manifest writer may stamp the same map into every
+ shard. There is **no locality constraint**: a shard may stamp tensors it
+ does not itself contain.
+- **No coverage requirement at load.** Unstamped tensors β€” and wholly
+ unstamped containers β€” load exactly as before this feature existed;
+ completeness of a stamping tool's coverage is a writer-side guarantee
+ (a load-time coverage diagnostic is deferred, not implied).
+
### Stamp-map scan bound
`st_fmt_stamp_ingest` (`c/st.h`) caps the total number of stamped-tensor