Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(xdna): decline the K1 planar int4 layout
Kenneth-Javier Ortigozae9cfdb09dbb2
diff --git a/c/backend_xdna.c b/c/backend_xdna.cindex 5c052615..76299a12 100644--- a/c/backend_xdna.c+++ b/c/backend_xdna.c@@ -918,6 +918,7 @@ const char *coli_xdna_hard_text(ColiXdnaHard h){case COLI_XDNA_HARD_SHAPE_UNSUPPORTED: return "SHAPE_UNSUPPORTED";case COLI_XDNA_HARD_FORMAT_UNSUPPORTED: return "FORMAT_UNSUPPORTED";case COLI_XDNA_HARD_GROUP_SIZE_UNSUPPORTED: return "GROUP_SIZE_UNSUPPORTED";+ case COLI_XDNA_HARD_LAYOUT_UNSUPPORTED: return "LAYOUT_UNSUPPORTED";case COLI_XDNA_HARD_ARTIFACT_UNAVAILABLE: return "ARTIFACT_UNAVAILABLE";case COLI_XDNA_HARD_ARTIFACT_INTEGRITY_FAILED: return "ARTIFACT_INTEGRITY_FAILED";case COLI_XDNA_HARD_ARTIFACT_UNQUALIFIED: return "ARTIFACT_UNQUALIFIED";@@ -1054,7 +1055,7 @@ int coli_xdna_test_conversions(void){ return g_xdna_conversions; }* returns, none of them merely records. */static ColiXdnaHard coli_xdna_engine_gates(ColiXdnaFamily family,- int fmt, int I, int O, int gs, int S,+ int fmt, int I, int O, int gs, int planar, int S,const ColiXdnaArtifact **row_out){/* 1 semantic family -- passed in by the call site, never inferred */if(family != COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP)@@ -1074,6 +1075,19 @@ static ColiXdnaHard coli_xdna_engine_gates(ColiXdnaFamily family,if(gs != COLI_XDNA_QUALIFIED_GROUP_SIZE)return COLI_XDNA_HARD_GROUP_SIZE_UNSUPPORTED;+ /* 5b in-memory byte layout.+ *+ * fmt=4 has TWO layouts in current Colibri. The classic PAIR layout+ * puts elements 2j and 2j+1 in byte j; the K1 PLANAR layout puts+ * elements k and k+32 in byte k of each 64-element block, and+ * qt_planarize() rewrites the tensor IN PLACE when the grouped planar+ * IDOT path is opted into (IDOT_GS=1), for exactly the gs>=64 tensors+ * this lane qualifies. The converter was qualified against the pair+ * layout only, so planar bytes would decode to nonsense rather than+ * fail. Decline instead. */+ if(planar)+ return COLI_XDNA_HARD_LAYOUT_UNSUPPORTED;+/* 6 a qualified artifact for exactly this family/bucket/shape/dtypes.* The bucket is the artifact's M, which is where logical M is mapped. */ColiXdnaRequest q;@@ -1137,7 +1151,7 @@ static int coli_xdna_stage(void **base, size_t *cap, size_t need){static ColiXdnaExec coli_xdna_attempt(ColiXdnaFamily family,ColiXdnaPrepared **slot,int fmt, const unsigned char *q4, const float *scale,- int I, int O, int gs,+ int I, int O, int gs, int planar,float *y, const float *x, int S){g_xdna_output_valid = 0; /* nothing is valid until completion says so */@@ -1157,7 +1171,7 @@ static ColiXdnaExec coli_xdna_attempt(ColiXdnaFamily family,}const ColiXdnaArtifact *row = NULL;- ColiXdnaHard h = coli_xdna_engine_gates(family, fmt, I, O, gs, S, &row);+ ColiXdnaHard h = coli_xdna_engine_gates(family, fmt, I, O, gs, planar, S, &row);if(h != COLI_XDNA_HARD_ELIGIBLE){g_xdna_last_hard = h;return (g_xdna_last_exec = COLI_XDNA_EXEC_DECLINED);@@ -1310,7 +1324,7 @@ static ColiXdnaExec coli_xdna_attempt(ColiXdnaFamily family,int coli_xdna_try_matmul(ColiXdnaFamily family,ColiXdnaPrepared **slot,int fmt, const unsigned char *q4, const float *scale,- int I, int O, int gs,+ int I, int O, int gs, int planar,float *y, const float *x, int S){/* Gate 0: the internal control. Without it this function is inert, which is@@ -1324,7 +1338,7 @@ int coli_xdna_try_matmul(ColiXdnaFamily family,g_xdna_last_exec = COLI_XDNA_EXEC_DECLINED;return 0;}- if(coli_xdna_attempt(family, slot, fmt, q4, scale, I, O, gs, y, x, S)+ if(coli_xdna_attempt(family, slot, fmt, q4, scale, I, O, gs, planar, y, x, S)== COLI_XDNA_EXEC_OK)return 1;g_xdna_fallbacks++;@@ -1340,10 +1354,10 @@ int coli_xdna_try_matmul(ColiXdnaFamily family,ColiXdnaExec coli_xdna_test_attempt(ColiXdnaFamily family,ColiXdnaPrepared **slot,int fmt, const unsigned char *q4, const float *scale,- int I, int O, int gs,+ int I, int O, int gs, int planar,float *y, const float *x, int S){- return coli_xdna_attempt(family, slot, fmt, q4, scale, I, O, gs, y, x, S);+ return coli_xdna_attempt(family, slot, fmt, q4, scale, I, O, gs, planar, y, x, S);}int coli_xdna_test_output_valid(void){ return g_xdna_output_valid; }diff --git a/c/backend_xdna.h b/c/backend_xdna.hindex efc6f127..915256e0 100644--- a/c/backend_xdna.h+++ b/c/backend_xdna.h@@ -359,6 +359,7 @@ typedef enum {COLI_XDNA_HARD_SHAPE_UNSUPPORTED,COLI_XDNA_HARD_FORMAT_UNSUPPORTED, /* stored tensor is not fmt=4 */COLI_XDNA_HARD_GROUP_SIZE_UNSUPPORTED, /* fmt=4 but gs is not the qualified 64 */+ COLI_XDNA_HARD_LAYOUT_UNSUPPORTED, /* fmt=4 pair layout expected; bytes are planar *//* artifact gates -- kept distinct because they mean different things* operationally: "this build does not ship that artifact", "the bytes are* not the bytes that were qualified", and "we never qualified this one"@@ -458,7 +459,7 @@ const char *coli_xdna_exec_text(ColiXdnaExec e);int coli_xdna_try_matmul(ColiXdnaFamily family,ColiXdnaPrepared **slot,int fmt, const unsigned char *q4, const float *scale,- int I, int O, int gs,+ int I, int O, int gs, int planar,float *y, const float *x, int S);/* Release helper-owned runtime state and the transient staging buffers. Safe@@ -486,7 +487,7 @@ void coli_xdna_execution_shutdown(void);ColiXdnaExec coli_xdna_test_attempt(ColiXdnaFamily family,ColiXdnaPrepared **slot,int fmt, const unsigned char *q4, const float *scale,- int I, int O, int gs,+ int I, int O, int gs, int planar,float *y, const float *x, int S);/* 1 only after an attempt reported successful completion. Any failure -- at anydiff --git a/c/colibri.c b/c/colibri.cindex 454b4c8c..ec625d21 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -6255,7 +6255,8 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int#ifdef COLI_XDNAif(!coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, &l->sh_gate.xdna,l->sh_gate.fmt, l->sh_gate.q4, l->sh_gate.s,- l->sh_gate.I, l->sh_gate.O, l->sh_gate.gs, sg, x, S))+ l->sh_gate.I, l->sh_gate.O, l->sh_gate.gs, l->sh_gate.planar,+ sg, x, S))#endif#ifdef COLI_VULKANif(!vk_matmul_qt(&l->sh_gate, sg, x, S))@@ -6264,7 +6265,8 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int#ifdef COLI_XDNAif(!coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, &l->sh_up.xdna,l->sh_up.fmt, l->sh_up.q4, l->sh_up.s,- l->sh_up.I, l->sh_up.O, l->sh_up.gs, su, x, S))+ l->sh_up.I, l->sh_up.O, l->sh_up.gs, l->sh_up.planar,+ su, x, S))#endif#ifdef COLI_VULKANif(!vk_matmul_qt(&l->sh_up, su, x, S))diff --git a/c/tests/test_xdna_execution.c b/c/tests/test_xdna_execution.cindex f4b569a0..ae837ab7 100644--- a/c/tests/test_xdna_execution.c+++ b/c/tests/test_xdna_execution.c@@ -179,7 +179,7 @@ int main(int argc, char **argv){float *x = mk_x(TM, TK, 1u);float *y = (float *)malloc((size_t)TM*TN*4); poison(y,(size_t)TM*TN);int handled = coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, &slot,- 4, W.q4, W.s, TK, TN, 64, y, x, TM);+ 4, W.q4, W.s, TK, TN, 64, 0, y, x, TM);ck(handled == 0, "helper, artifact and device all available -- still not handled");ck(coli_xdna_test_dispatches() == 0, "and zero dispatches");ck(all_poison(y,(size_t)TM*TN), "caller output untouched");@@ -196,7 +196,7 @@ int main(int argc, char **argv){float *x = mk_x(TM, TK, 2u);float *y = (float *)malloc((size_t)TM*TN*4); poison(y,(size_t)TM*TN);int handled = coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, &slot,- 4, W.q4, W.s, TK, TN, 64, y, x, TM);+ 4, W.q4, W.s, TK, TN, 64, 0, y, x, TM);ck(handled == 1, "handled");ck(coli_xdna_test_last_hard() == COLI_XDNA_HARD_ELIGIBLE, "FULL_XDNA_HARD_ELIGIBLE");ck(coli_xdna_test_last_exec() == COLI_XDNA_EXEC_OK, "exec OK");@@ -230,7 +230,7 @@ int main(int argc, char **argv){float *x = mk_x(S, TK, 100u + (unsigned)S);float *y = (float *)malloc((size_t)S*TN*4); poison(y,(size_t)S*TN);int handled = coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, &slot,- 4, W.q4, W.s, TK, TN, 64, y, x, S);+ 4, W.q4, W.s, TK, TN, 64, 0, y, x, S);char msg[128];snprintf(msg,sizeof msg,"S=%d handled",S); ck(handled==1,msg);snprintf(msg,sizeof msg,"S=%d padded_ops=%d",S,coli_xdna_test_padded_operations());@@ -277,7 +277,7 @@ int main(int argc, char **argv){float *y = (float *)malloc((size_t)S*cases[c].O*4); poison(y,(size_t)S*cases[c].O);int handled = coli_xdna_try_matmul(cases[c].fam, &slot, cases[c].fmt,T.q4, T.s, cases[c].I, cases[c].O,- cases[c].gs, y, x, cases[c].S);+ cases[c].gs, 0, y, x, cases[c].S);char msg[160];snprintf(msg,sizeof msg,"%s -> not handled", cases[c].what); ck(handled==0,msg);snprintf(msg,sizeof msg,"%s -> dispatches 0", cases[c].what);@@ -310,7 +310,7 @@ int main(int argc, char **argv){float *x = mk_x(TM, TK, 4u);float *y = (float *)malloc((size_t)TM*TN*4); poison(y,(size_t)TM*TN);ck(coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&slot,4,T.q4,T.s,- TK,TN,32,y,x,TM) == 0,+ TK,TN,32,0,y,x,TM) == 0,"GENERIC_PREPARABLE is not INITIAL_XDNA_QUALIFIED");ck(coli_xdna_test_dispatches()==0, "dispatches 0");coli_xdna_prepared_release(&slot); free(x); free(y); fmt4_free(&T);@@ -338,7 +338,7 @@ int main(int argc, char **argv){float *x = mk_x(TM, TK, 5u);float *y = (float *)malloc((size_t)TM*TN*4); poison(y,(size_t)TM*TN);int handled = coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&slot,- 4,W.q4,W.s,TK,TN,64,y,x,TM);+ 4,W.q4,W.s,TK,TN,64,0,y,x,TM);char msg[160];snprintf(msg,sizeof msg,"%s -> not handled", hs[c].what); ck(handled==0,msg);snprintf(msg,sizeof msg,"%s -> dispatches 0", hs[c].what);@@ -373,7 +373,7 @@ int main(int argc, char **argv){float *x = mk_x(TM, TK, 6u);float *y = (float *)malloc((size_t)TM*TN*4); poison(y,(size_t)TM*TN);int handled = coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&slot,- 4,W.q4,W.s,TK,TN,64,y,x,TM);+ 4,W.q4,W.s,TK,TN,64,0,y,x,TM);const char *what = c==0 ? "artifact absent" : "artifact hash mismatch";/* Since W2-N7-I6 these are DIFFERENT verdicts: a build that does not* ship the bytes and a build whose bytes were tampered with call for@@ -424,7 +424,7 @@ int main(int argc, char **argv){float *x = mk_x(TM, TK, 7u);float *y = (float *)malloc((size_t)TM*TN*4); poison(y,(size_t)TM*TN);int handled = coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&slot,- 4,W.q4,W.s,TK,TN,64,y,x,TM);+ 4,W.q4,W.s,TK,TN,64,0,y,x,TM);char msg[200];snprintf(msg,sizeof msg,"%s failure -> not handled", fs[c].what); ck(handled==0,msg);snprintf(msg,sizeof msg,"%s failure -> %s", fs[c].what,@@ -459,16 +459,16 @@ int main(int argc, char **argv){float *ya2= (float *)malloc((size_t)TM*TN*4);ck(coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&sa,4,W.q4,W.s,- TK,TN,64,ya,x,TM)==1, "gate-like weight executes");+ TK,TN,64,0,ya,x,TM)==1, "gate-like weight executes");ck(coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&sa,4,W.q4,W.s,- TK,TN,64,ya2,x,TM)==1, "same weight again executes");+ TK,TN,64,0,ya2,x,TM)==1, "same weight again executes");ck(coli_xdna_test_conversions()-conv0 == 1, "prepared once, reused on the second call");ck(coli_xdna_test_userptr_wraps() == 1, "wrapped once for the same pointer");ck(coli_xdna_test_artifact_opens() == 1, "artifact runtime opened once, not per operation");ck(memcmp(ya,ya2,(size_t)TM*TN*4)==0, "identical inputs give identical outputs");ck(coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&sb,4,W2.q4,W2.s,- TK,TN,64,yb,x,TM)==1, "up-like weight executes");+ TK,TN,64,0,yb,x,TM)==1, "up-like weight executes");ck(coli_xdna_test_userptr_wraps() == 2, "a different prepared image is wrapped again");ck(coli_xdna_test_artifact_opens() == 1, "and reuses the same artifact runtime");ck(memcmp(ya,yb,(size_t)TM*TN*4)!=0, "different runtime weights give different outputs");@@ -495,7 +495,7 @@ int main(int argc, char **argv){float *x = mk_x(TM,TK,10u);float *y = (float*)malloc((size_t)TM*TN*4);ck(coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&slot,4,W.q4,W.s,- TK,TN,64,y,x,TM)==1, "operation succeeds");+ TK,TN,64,0,y,x,TM)==1, "operation succeeds");coli_xdna_execution_shutdown();ck(1, "shutdown after a successful operation is safe");coli_xdna_execution_shutdown();diff --git a/c/tests/test_xdna_failure.c b/c/tests/test_xdna_failure.cindex 55e1343f..2f0f5300 100644--- a/c/tests/test_xdna_failure.c+++ b/c/tests/test_xdna_failure.c@@ -158,7 +158,7 @@ static void use_helper(const char *path){* existed. Nothing else is a fallback. */static void seam(float *y, const float *x, QT *w, int S){if(!coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, &w->xdna,- w->fmt, w->q4, w->s, w->I, w->O, w->gs, y, x, S))+ w->fmt, w->q4, w->s, w->I, w->O, w->gs, w->planar, y, x, S))matmul_qt(y, x, w, S);}@@ -219,13 +219,15 @@ int main(int argc, char **argv){/* ================================================================== */printf("\npre-device declines -- nothing may reach the device\n");{- struct { const char *what; ColiXdnaFamily fam; int fmt,I,O,gs,S; ColiXdnaHard want; } c[] = {- { "wrong semantic family", COLI_XDNA_FAMILY_NONE, 4,TK,TN,64,TM, COLI_XDNA_HARD_FAMILY_UNSUPPORTED },- { "sh_down orientation", COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, 4,TN,TK,64,TM, COLI_XDNA_HARD_SHAPE_UNSUPPORTED },- { "gs != 64", COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, 4,TK,TN,32,TM, COLI_XDNA_HARD_GROUP_SIZE_UNSUPPORTED },- { "fmt != 4", COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, 2,TK,TN,64,TM, COLI_XDNA_HARD_FORMAT_UNSUPPORTED },- { "M above range", COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, 4,TK,TN,64,TM+1, COLI_XDNA_HARD_M_OUT_OF_RANGE },- { "K/N mismatch", COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, 4,TK+8,TN,64,TM, COLI_XDNA_HARD_SHAPE_UNSUPPORTED }+ struct { const char *what; ColiXdnaFamily fam; int fmt,I,O,gs,planar,S; ColiXdnaHard want; } c[] = {+ { "wrong semantic family", COLI_XDNA_FAMILY_NONE, 4,TK,TN,64,0,TM, COLI_XDNA_HARD_FAMILY_UNSUPPORTED },+ { "sh_down orientation", COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, 4,TN,TK,64,0,TM, COLI_XDNA_HARD_SHAPE_UNSUPPORTED },+ { "gs != 64", COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, 4,TK,TN,32,0,TM, COLI_XDNA_HARD_GROUP_SIZE_UNSUPPORTED },+ { "fmt != 4", COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, 2,TK,TN,64,0,TM, COLI_XDNA_HARD_FORMAT_UNSUPPORTED },+ { "M above range", COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, 4,TK,TN,64,0,TM+1, COLI_XDNA_HARD_M_OUT_OF_RANGE },+ { "K/N mismatch", COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, 4,TK+8,TN,64,0,TM, COLI_XDNA_HARD_SHAPE_UNSUPPORTED },+ /* K1 planar layout: the converter was never qualified against it */+ { "planar fmt4 layout", COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, 4,TK,TN,64,1,TM, COLI_XDNA_HARD_LAYOUT_UNSUPPORTED }};for(size_t i = 0; i < sizeof c/sizeof c[0]; i++){use_helper(g_helper);@@ -239,7 +241,7 @@ int main(int argc, char **argv){T.fmt = 4; matmul_qt(rr, xx, &T, c[i].S); T.fmt = c[i].fmt;poison_buf(yy,(size_t)c[i].S*c[i].O);int handled = coli_xdna_try_matmul(c[i].fam, &T.xdna, c[i].fmt, T.q4, T.s,- c[i].I, c[i].O, c[i].gs, yy, xx, c[i].S);+ c[i].I, c[i].O, c[i].gs, c[i].planar, yy, xx, c[i].S);char m[200];snprintf(m,sizeof m,"%s -> %s", c[i].what, coli_xdna_hard_text(coli_xdna_test_last_hard()));ck(coli_xdna_test_last_hard()==c[i].want, m);@@ -266,7 +268,7 @@ int main(int argc, char **argv){char none[1200]; snprintf(none,sizeof none,"%s/absent", g_root);coli_xdna_test_set_artifact_root(none);float *y = (float*)malloc((size_t)TM*TN*4); poison_buf(y,(size_t)TM*TN);- coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&W.xdna,4,W.q4,W.s,TK,TN,64,y,x,TM);+ coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&W.xdna,4,W.q4,W.s,TK,TN,64,0,y,x,TM);ck(coli_xdna_test_last_hard()==COLI_XDNA_HARD_ARTIFACT_UNAVAILABLE,"artifact absent -> ARTIFACT_UNAVAILABLE");ck(coli_xdna_test_helper_calls()==0, "artifact absent -> helper never called");@@ -274,7 +276,7 @@ int main(int argc, char **argv){coli_xdna_test_set_artifact_root(g_root);{ FILE *f = fopen(g_xclbin,"r+b"); if(f){ fseek(f,64,SEEK_SET); fputc(0x5A,f); fclose(f); } }- coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&W.xdna,4,W.q4,W.s,TK,TN,64,y,x,TM);+ coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&W.xdna,4,W.q4,W.s,TK,TN,64,0,y,x,TM);ck(coli_xdna_test_last_hard()==COLI_XDNA_HARD_ARTIFACT_INTEGRITY_FAILED,"artifact tampered -> ARTIFACT_INTEGRITY_FAILED (not the same as absent)");ck(coli_xdna_test_helper_calls()==0, "artifact tampered -> unverified bytes never reach the helper");@@ -282,7 +284,7 @@ int main(int argc, char **argv){write_blob(g_xclbin, 31u, 4096); build_registry();g_test_rows[0].correctness_qualified = 0;- coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&W.xdna,4,W.q4,W.s,TK,TN,64,y,x,TM);+ coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&W.xdna,4,W.q4,W.s,TK,TN,64,0,y,x,TM);ck(coli_xdna_test_last_hard()==COLI_XDNA_HARD_ARTIFACT_UNQUALIFIED,"artifact never correctness-qualified -> ARTIFACT_UNQUALIFIED");g_test_rows[0].correctness_qualified = 1;@@ -303,7 +305,7 @@ int main(int argc, char **argv){coli_xdna_test_set_force_execution(1);QT T; qt_make(&T, TK, TN, 64, 4242u);float *y = (float*)malloc((size_t)TM*TN*4); poison_buf(y,(size_t)TM*TN);- coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&T.xdna,4,T.q4,T.s,TK,TN,64,y,x,TM);+ coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&T.xdna,4,T.q4,T.s,TK,TN,64,0,y,x,TM);char m[200];snprintf(m,sizeof m,"%s -> %s", h[i].what, coli_xdna_hard_text(coli_xdna_test_last_hard()));ck(coli_xdna_test_last_hard()==h[i].want, m);@@ -332,7 +334,7 @@ int main(int argc, char **argv){coli_xdna_test_set_convert_fail_pct(50);float *y = (float*)malloc((size_t)TM*TN*4); poison_buf(y,(size_t)TM*TN);int handled = coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&T.xdna,- 4,T.q4,T.s,TK,TN,64,y,x,TM);+ 4,T.q4,T.s,TK,TN,64,0,y,x,TM);ck(handled==0, "mid-conversion failure -> not handled");ck(coli_xdna_test_last_exec()==COLI_XDNA_EXEC_WEIGHT_PREPARE_FAILED,"classified WEIGHT_PREPARE_FAILED");@@ -351,7 +353,7 @@ int main(int argc, char **argv){coli_xdna_test_set_convert_fail_pct(0);poison_buf(y,(size_t)TM*TN);ck(coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&T.xdna,- 4,T.q4,T.s,TK,TN,64,y,x,TM)==1,+ 4,T.q4,T.s,TK,TN,64,0,y,x,TM)==1,"a complete re-preparation recovers and executes");ck(coli_xdna_prepared_state(T.xdna)==COLI_XDNA_PREP_VALID, "state VALID again");ck(coli_xdna_test_output_valid()==1, "and the output is valid");@@ -378,7 +380,7 @@ int main(int argc, char **argv){float *y = (float*)malloc((size_t)TM*TN*4); poison_buf(y,(size_t)TM*TN);int d0 = coli_xdna_test_dispatches();int handled = coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&T.xdna,- 4,T.q4,T.s,TK,TN,64,y,x,TM);+ 4,T.q4,T.s,TK,TN,64,0,y,x,TM);char m[220];snprintf(m,sizeof m,"%s failure -> not handled", st[i].what); ck(handled==0,m);snprintf(m,sizeof m,"%s failure -> %s", st[i].what,@@ -423,7 +425,7 @@ int main(int argc, char **argv){int d_late = coli_xdna_test_dispatches();int handled = coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&T.xdna,- 4,T.q4,T.s,TK,TN,64,y,x,TM);+ 4,T.q4,T.s,TK,TN,64,0,y,x,TM);ck(handled==0, "not handled");ck(coli_xdna_test_last_exec()==COLI_XDNA_EXEC_COMPLETION_FAILED, "COMPLETION_FAILED");@@ -463,7 +465,7 @@ int main(int argc, char **argv){QT T; qt_make(&T, TK, TN, 64, 4242u);float *y = (float*)malloc((size_t)TM*TN*4); poison_buf(y,(size_t)TM*TN);ColiXdnaExec e = coli_xdna_test_attempt(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&T.xdna,- 4,T.q4,T.s,TK,TN,64,y,x,TM);+ 4,T.q4,T.s,TK,TN,64,0,y,x,TM);ck(e==COLI_XDNA_EXEC_EXECUTE_FAILED, "explicit mode returns the failure class");ck(coli_xdna_test_output_valid()==0, "output not valid");ck(count_poison(y,(size_t)TM*TN)==(size_t)TM*TN,@@ -471,7 +473,7 @@ int main(int argc, char **argv){arm(F_NONE);poison_buf(y,(size_t)TM*TN);e = coli_xdna_test_attempt(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&T.xdna,- 4,T.q4,T.s,TK,TN,64,y,x,TM);+ 4,T.q4,T.s,TK,TN,64,0,y,x,TM);ck(e==COLI_XDNA_EXEC_OK, "explicit mode succeeds when the lane works");ck(coli_xdna_test_output_valid()==1, "and the output is valid");ck(count_poison(y,(size_t)TM*TN)==0, "and was written");@@ -495,11 +497,11 @@ int main(int argc, char **argv){arm(F_WRAP);QT T; qt_make(&T, TK, TN, 64, 4242u);float *y = (float*)malloc((size_t)TM*TN*4);- coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&T.xdna,4,T.q4,T.s,TK,TN,64,y,x,TM);+ coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&T.xdna,4,T.q4,T.s,TK,TN,64,0,y,x,TM);ck(coli_xdna_lane_health()==COLI_XDNA_LANE_HEALTHY,"a wrap failure leaves the lane HEALTHY -- it is one operation");arm(F_NONE);- ck(coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&T.xdna,4,T.q4,T.s,TK,TN,64,y,x,TM)==1,+ ck(coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&T.xdna,4,T.q4,T.s,TK,TN,64,0,y,x,TM)==1,"and the very next operation succeeds");use_helper(g_helper);@@ -507,12 +509,12 @@ int main(int argc, char **argv){coli_xdna_test_set_force_execution(1);arm(F_DEVICE);QT D; qt_make(&D, TK, TN, 64, 4242u);- coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&D.xdna,4,D.q4,D.s,TK,TN,64,y,x,TM);+ coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&D.xdna,4,D.q4,D.s,TK,TN,64,0,y,x,TM);ck(coli_xdna_lane_health()==COLI_XDNA_LANE_UNAVAILABLE,"a device-init failure marks the lane UNAVAILABLE for the process");arm(F_NONE);int o0 = coli_xdna_test_device_opens();- coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&D.xdna,4,D.q4,D.s,TK,TN,64,y,x,TM);+ coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&D.xdna,4,D.q4,D.s,TK,TN,64,0,y,x,TM);ck(coli_xdna_test_device_opens()==o0,"and the device is not re-attempted per operation");ck(coli_xdna_test_last_hard()==COLI_XDNA_HARD_DEVICE_UNAVAILABLE, "the decline says why");diff --git a/c/tests/xdna_physical_probe.c b/c/tests/xdna_physical_probe.cindex 62a68318..f1bc0e08 100644--- a/c/tests/xdna_physical_probe.c+++ b/c/tests/xdna_physical_probe.c@@ -188,7 +188,7 @@ int main(int argc, char **argv){double t0 = now_s();int handled = coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&w->xdna, w->fmt, w->q4, w->s,- w->I, w->O, w->gs, y, x, S);+ w->I, w->O, w->gs, w->planar, y, x, S);double ms = (now_s()-t0)*1000.0;printf("\n-- %s M=%d --\n", names[t], S);@@ -278,7 +278,7 @@ int main(int argc, char **argv){float *y = (float*)malloc((size_t)S*PN*4);int handled = coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&G.xdna, G.fmt, G.q4, G.s,- G.I, G.O, G.gs, y, x, S);+ G.I, G.O, G.gs, G.planar, y, x, S);printf("\n-- generic f32 activations (BF16 rounding is LOSSY here), M=%d --\n", S);printf("HANDLED %d\n", handled);if(!handled) g_bad = 1;@@ -323,14 +323,14 @@ int main(int argc, char **argv){float *yb = (float*)malloc((size_t)S*PN*4);printf("\n-- stale-wrapper control: reprepare a different weight in place --\n");int ha = coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, &G.xdna,- G.fmt, G.q4, G.s, G.I, G.O, G.gs, ya, x, S);+ G.fmt, G.q4, G.s, G.I, G.O, G.gs, G.planar, ya, x, S);const void *p1 = coli_xdna_prepared_image(G.xdna);int w0 = coli_xdna_test_userptr_wraps();coli_xdna_prepared_invalidate(G.xdna);ColiXdnaPrepResult pr = coli_xdna_prepare_from_fmt4(G.xdna, 4, U.q4, U.s, PK, PN, PGS);const void *p2 = coli_xdna_prepared_image(G.xdna);int hb = coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, &G.xdna,- G.fmt, G.q4, G.s, G.I, G.O, G.gs, yb, x, S);+ G.fmt, G.q4, G.s, G.I, G.O, G.gs, G.planar, yb, x, S);printf("PREPARE_RC %d ptr_same %s\n", (int)pr, p1==p2?"YES":"no");printf("HANDLED %d %d\n", ha, hb);printf("WRAPS before=%d after=%d rewrap=%s\n", w0,@@ -362,7 +362,7 @@ int main(int argc, char **argv){float *y = (float*)malloc((size_t)8*PN*4);int before = coli_xdna_test_dispatches();int h2 = coli_xdna_try_matmul(COLI_XDNA_FAMILY_NONE, &G.xdna, G.fmt, G.q4, G.s,- G.I, G.O, G.gs, y, x, 8);+ G.I, G.O, G.gs, G.planar, y, x, 8);printf("\nWRONG_FAMILY_HANDLED %d dispatch_delta %d (%s)\n", h2,coli_xdna_test_dispatches()-before,coli_xdna_hard_text(coli_xdna_test_last_hard()));