Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(xdna): decline the K1 planar int4 layout

Kenneth-Javier Ortigozae9cfdb09dbb2
diff --git a/c/backend_xdna.c b/c/backend_xdna.c
index 5c052615..76299a12 100644
--- a/c/backend_xdna.c
+++ b/c/backend_xdna.c
@@ -918,6 +918,7 @@ const char *coli_xdna_hard_text(ColiXdnaHard h){
case COLI_XDNA_HARD_SHAPE_UNSUPPORTED: return "SHAPE_UNSUPPORTED";
case COLI_XDNA_HARD_FORMAT_UNSUPPORTED: return "FORMAT_UNSUPPORTED";
case COLI_XDNA_HARD_GROUP_SIZE_UNSUPPORTED: return "GROUP_SIZE_UNSUPPORTED";
+ case COLI_XDNA_HARD_LAYOUT_UNSUPPORTED: return "LAYOUT_UNSUPPORTED";
case COLI_XDNA_HARD_ARTIFACT_UNAVAILABLE: return "ARTIFACT_UNAVAILABLE";
case COLI_XDNA_HARD_ARTIFACT_INTEGRITY_FAILED: return "ARTIFACT_INTEGRITY_FAILED";
case COLI_XDNA_HARD_ARTIFACT_UNQUALIFIED: return "ARTIFACT_UNQUALIFIED";
@@ -1054,7 +1055,7 @@ int coli_xdna_test_conversions(void){ return g_xdna_conversions; }
* returns, none of them merely records. */
static ColiXdnaHard coli_xdna_engine_gates(ColiXdnaFamily family,
- int fmt, int I, int O, int gs, int S,
+ int fmt, int I, int O, int gs, int planar, int S,
const ColiXdnaArtifact **row_out){
/* 1 semantic family -- passed in by the call site, never inferred */
if(family != COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP)
@@ -1074,6 +1075,19 @@ static ColiXdnaHard coli_xdna_engine_gates(ColiXdnaFamily family,
if(gs != COLI_XDNA_QUALIFIED_GROUP_SIZE)
return COLI_XDNA_HARD_GROUP_SIZE_UNSUPPORTED;
+ /* 5b in-memory byte layout.
+ *
+ * fmt=4 has TWO layouts in current Colibri. The classic PAIR layout
+ * puts elements 2j and 2j+1 in byte j; the K1 PLANAR layout puts
+ * elements k and k+32 in byte k of each 64-element block, and
+ * qt_planarize() rewrites the tensor IN PLACE when the grouped planar
+ * IDOT path is opted into (IDOT_GS=1), for exactly the gs>=64 tensors
+ * this lane qualifies. The converter was qualified against the pair
+ * layout only, so planar bytes would decode to nonsense rather than
+ * fail. Decline instead. */
+ if(planar)
+ return COLI_XDNA_HARD_LAYOUT_UNSUPPORTED;
+
/* 6 a qualified artifact for exactly this family/bucket/shape/dtypes.
* The bucket is the artifact's M, which is where logical M is mapped. */
ColiXdnaRequest q;
@@ -1137,7 +1151,7 @@ static int coli_xdna_stage(void **base, size_t *cap, size_t need){
static ColiXdnaExec coli_xdna_attempt(ColiXdnaFamily family,
ColiXdnaPrepared **slot,
int fmt, const unsigned char *q4, const float *scale,
- int I, int O, int gs,
+ int I, int O, int gs, int planar,
float *y, const float *x, int S)
{
g_xdna_output_valid = 0; /* nothing is valid until completion says so */
@@ -1157,7 +1171,7 @@ static ColiXdnaExec coli_xdna_attempt(ColiXdnaFamily family,
}
const ColiXdnaArtifact *row = NULL;
- ColiXdnaHard h = coli_xdna_engine_gates(family, fmt, I, O, gs, S, &row);
+ ColiXdnaHard h = coli_xdna_engine_gates(family, fmt, I, O, gs, planar, S, &row);
if(h != COLI_XDNA_HARD_ELIGIBLE){
g_xdna_last_hard = h;
return (g_xdna_last_exec = COLI_XDNA_EXEC_DECLINED);
@@ -1310,7 +1324,7 @@ static ColiXdnaExec coli_xdna_attempt(ColiXdnaFamily family,
int coli_xdna_try_matmul(ColiXdnaFamily family,
ColiXdnaPrepared **slot,
int fmt, const unsigned char *q4, const float *scale,
- int I, int O, int gs,
+ int I, int O, int gs, int planar,
float *y, const float *x, int S)
{
/* Gate 0: the internal control. Without it this function is inert, which is
@@ -1324,7 +1338,7 @@ int coli_xdna_try_matmul(ColiXdnaFamily family,
g_xdna_last_exec = COLI_XDNA_EXEC_DECLINED;
return 0;
}
- if(coli_xdna_attempt(family, slot, fmt, q4, scale, I, O, gs, y, x, S)
+ if(coli_xdna_attempt(family, slot, fmt, q4, scale, I, O, gs, planar, y, x, S)
== COLI_XDNA_EXEC_OK)
return 1;
g_xdna_fallbacks++;
@@ -1340,10 +1354,10 @@ int coli_xdna_try_matmul(ColiXdnaFamily family,
ColiXdnaExec coli_xdna_test_attempt(ColiXdnaFamily family,
ColiXdnaPrepared **slot,
int fmt, const unsigned char *q4, const float *scale,
- int I, int O, int gs,
+ int I, int O, int gs, int planar,
float *y, const float *x, int S)
{
- return coli_xdna_attempt(family, slot, fmt, q4, scale, I, O, gs, y, x, S);
+ return coli_xdna_attempt(family, slot, fmt, q4, scale, I, O, gs, planar, y, x, S);
}
int coli_xdna_test_output_valid(void){ return g_xdna_output_valid; }
diff --git a/c/backend_xdna.h b/c/backend_xdna.h
index efc6f127..915256e0 100644
--- a/c/backend_xdna.h
+++ b/c/backend_xdna.h
@@ -359,6 +359,7 @@ typedef enum {
COLI_XDNA_HARD_SHAPE_UNSUPPORTED,
COLI_XDNA_HARD_FORMAT_UNSUPPORTED, /* stored tensor is not fmt=4 */
COLI_XDNA_HARD_GROUP_SIZE_UNSUPPORTED, /* fmt=4 but gs is not the qualified 64 */
+ COLI_XDNA_HARD_LAYOUT_UNSUPPORTED, /* fmt=4 pair layout expected; bytes are planar */
/* artifact gates -- kept distinct because they mean different things
* operationally: "this build does not ship that artifact", "the bytes are
* not the bytes that were qualified", and "we never qualified this one"
@@ -458,7 +459,7 @@ const char *coli_xdna_exec_text(ColiXdnaExec e);
int coli_xdna_try_matmul(ColiXdnaFamily family,
ColiXdnaPrepared **slot,
int fmt, const unsigned char *q4, const float *scale,
- int I, int O, int gs,
+ int I, int O, int gs, int planar,
float *y, const float *x, int S);
/* Release helper-owned runtime state and the transient staging buffers. Safe
@@ -486,7 +487,7 @@ void coli_xdna_execution_shutdown(void);
ColiXdnaExec coli_xdna_test_attempt(ColiXdnaFamily family,
ColiXdnaPrepared **slot,
int fmt, const unsigned char *q4, const float *scale,
- int I, int O, int gs,
+ int I, int O, int gs, int planar,
float *y, const float *x, int S);
/* 1 only after an attempt reported successful completion. Any failure -- at any
diff --git a/c/colibri.c b/c/colibri.c
index 454b4c8c..ec625d21 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -6255,7 +6255,8 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
#ifdef COLI_XDNA
if(!coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, &l->sh_gate.xdna,
l->sh_gate.fmt, l->sh_gate.q4, l->sh_gate.s,
- l->sh_gate.I, l->sh_gate.O, l->sh_gate.gs, sg, x, S))
+ l->sh_gate.I, l->sh_gate.O, l->sh_gate.gs, l->sh_gate.planar,
+ sg, x, S))
#endif
#ifdef COLI_VULKAN
if(!vk_matmul_qt(&l->sh_gate, sg, x, S))
@@ -6264,7 +6265,8 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
#ifdef COLI_XDNA
if(!coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, &l->sh_up.xdna,
l->sh_up.fmt, l->sh_up.q4, l->sh_up.s,
- l->sh_up.I, l->sh_up.O, l->sh_up.gs, su, x, S))
+ l->sh_up.I, l->sh_up.O, l->sh_up.gs, l->sh_up.planar,
+ su, x, S))
#endif
#ifdef COLI_VULKAN
if(!vk_matmul_qt(&l->sh_up, su, x, S))
diff --git a/c/tests/test_xdna_execution.c b/c/tests/test_xdna_execution.c
index f4b569a0..ae837ab7 100644
--- a/c/tests/test_xdna_execution.c
+++ b/c/tests/test_xdna_execution.c
@@ -179,7 +179,7 @@ int main(int argc, char **argv){
float *x = mk_x(TM, TK, 1u);
float *y = (float *)malloc((size_t)TM*TN*4); poison(y,(size_t)TM*TN);
int handled = coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, &slot,
- 4, W.q4, W.s, TK, TN, 64, y, x, TM);
+ 4, W.q4, W.s, TK, TN, 64, 0, y, x, TM);
ck(handled == 0, "helper, artifact and device all available -- still not handled");
ck(coli_xdna_test_dispatches() == 0, "and zero dispatches");
ck(all_poison(y,(size_t)TM*TN), "caller output untouched");
@@ -196,7 +196,7 @@ int main(int argc, char **argv){
float *x = mk_x(TM, TK, 2u);
float *y = (float *)malloc((size_t)TM*TN*4); poison(y,(size_t)TM*TN);
int handled = coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, &slot,
- 4, W.q4, W.s, TK, TN, 64, y, x, TM);
+ 4, W.q4, W.s, TK, TN, 64, 0, y, x, TM);
ck(handled == 1, "handled");
ck(coli_xdna_test_last_hard() == COLI_XDNA_HARD_ELIGIBLE, "FULL_XDNA_HARD_ELIGIBLE");
ck(coli_xdna_test_last_exec() == COLI_XDNA_EXEC_OK, "exec OK");
@@ -230,7 +230,7 @@ int main(int argc, char **argv){
float *x = mk_x(S, TK, 100u + (unsigned)S);
float *y = (float *)malloc((size_t)S*TN*4); poison(y,(size_t)S*TN);
int handled = coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, &slot,
- 4, W.q4, W.s, TK, TN, 64, y, x, S);
+ 4, W.q4, W.s, TK, TN, 64, 0, y, x, S);
char msg[128];
snprintf(msg,sizeof msg,"S=%d handled",S); ck(handled==1,msg);
snprintf(msg,sizeof msg,"S=%d padded_ops=%d",S,coli_xdna_test_padded_operations());
@@ -277,7 +277,7 @@ int main(int argc, char **argv){
float *y = (float *)malloc((size_t)S*cases[c].O*4); poison(y,(size_t)S*cases[c].O);
int handled = coli_xdna_try_matmul(cases[c].fam, &slot, cases[c].fmt,
T.q4, T.s, cases[c].I, cases[c].O,
- cases[c].gs, y, x, cases[c].S);
+ cases[c].gs, 0, y, x, cases[c].S);
char msg[160];
snprintf(msg,sizeof msg,"%s -> not handled", cases[c].what); ck(handled==0,msg);
snprintf(msg,sizeof msg,"%s -> dispatches 0", cases[c].what);
@@ -310,7 +310,7 @@ int main(int argc, char **argv){
float *x = mk_x(TM, TK, 4u);
float *y = (float *)malloc((size_t)TM*TN*4); poison(y,(size_t)TM*TN);
ck(coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&slot,4,T.q4,T.s,
- TK,TN,32,y,x,TM) == 0,
+ TK,TN,32,0,y,x,TM) == 0,
"GENERIC_PREPARABLE is not INITIAL_XDNA_QUALIFIED");
ck(coli_xdna_test_dispatches()==0, "dispatches 0");
coli_xdna_prepared_release(&slot); free(x); free(y); fmt4_free(&T);
@@ -338,7 +338,7 @@ int main(int argc, char **argv){
float *x = mk_x(TM, TK, 5u);
float *y = (float *)malloc((size_t)TM*TN*4); poison(y,(size_t)TM*TN);
int handled = coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&slot,
- 4,W.q4,W.s,TK,TN,64,y,x,TM);
+ 4,W.q4,W.s,TK,TN,64,0,y,x,TM);
char msg[160];
snprintf(msg,sizeof msg,"%s -> not handled", hs[c].what); ck(handled==0,msg);
snprintf(msg,sizeof msg,"%s -> dispatches 0", hs[c].what);
@@ -373,7 +373,7 @@ int main(int argc, char **argv){
float *x = mk_x(TM, TK, 6u);
float *y = (float *)malloc((size_t)TM*TN*4); poison(y,(size_t)TM*TN);
int handled = coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&slot,
- 4,W.q4,W.s,TK,TN,64,y,x,TM);
+ 4,W.q4,W.s,TK,TN,64,0,y,x,TM);
const char *what = c==0 ? "artifact absent" : "artifact hash mismatch";
/* Since W2-N7-I6 these are DIFFERENT verdicts: a build that does not
* ship the bytes and a build whose bytes were tampered with call for
@@ -424,7 +424,7 @@ int main(int argc, char **argv){
float *x = mk_x(TM, TK, 7u);
float *y = (float *)malloc((size_t)TM*TN*4); poison(y,(size_t)TM*TN);
int handled = coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&slot,
- 4,W.q4,W.s,TK,TN,64,y,x,TM);
+ 4,W.q4,W.s,TK,TN,64,0,y,x,TM);
char msg[200];
snprintf(msg,sizeof msg,"%s failure -> not handled", fs[c].what); ck(handled==0,msg);
snprintf(msg,sizeof msg,"%s failure -> %s", fs[c].what,
@@ -459,16 +459,16 @@ int main(int argc, char **argv){
float *ya2= (float *)malloc((size_t)TM*TN*4);
ck(coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&sa,4,W.q4,W.s,
- TK,TN,64,ya,x,TM)==1, "gate-like weight executes");
+ TK,TN,64,0,ya,x,TM)==1, "gate-like weight executes");
ck(coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&sa,4,W.q4,W.s,
- TK,TN,64,ya2,x,TM)==1, "same weight again executes");
+ TK,TN,64,0,ya2,x,TM)==1, "same weight again executes");
ck(coli_xdna_test_conversions()-conv0 == 1, "prepared once, reused on the second call");
ck(coli_xdna_test_userptr_wraps() == 1, "wrapped once for the same pointer");
ck(coli_xdna_test_artifact_opens() == 1, "artifact runtime opened once, not per operation");
ck(memcmp(ya,ya2,(size_t)TM*TN*4)==0, "identical inputs give identical outputs");
ck(coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&sb,4,W2.q4,W2.s,
- TK,TN,64,yb,x,TM)==1, "up-like weight executes");
+ TK,TN,64,0,yb,x,TM)==1, "up-like weight executes");
ck(coli_xdna_test_userptr_wraps() == 2, "a different prepared image is wrapped again");
ck(coli_xdna_test_artifact_opens() == 1, "and reuses the same artifact runtime");
ck(memcmp(ya,yb,(size_t)TM*TN*4)!=0, "different runtime weights give different outputs");
@@ -495,7 +495,7 @@ int main(int argc, char **argv){
float *x = mk_x(TM,TK,10u);
float *y = (float*)malloc((size_t)TM*TN*4);
ck(coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&slot,4,W.q4,W.s,
- TK,TN,64,y,x,TM)==1, "operation succeeds");
+ TK,TN,64,0,y,x,TM)==1, "operation succeeds");
coli_xdna_execution_shutdown();
ck(1, "shutdown after a successful operation is safe");
coli_xdna_execution_shutdown();
diff --git a/c/tests/test_xdna_failure.c b/c/tests/test_xdna_failure.c
index 55e1343f..2f0f5300 100644
--- a/c/tests/test_xdna_failure.c
+++ b/c/tests/test_xdna_failure.c
@@ -158,7 +158,7 @@ static void use_helper(const char *path){
* existed. Nothing else is a fallback. */
static void seam(float *y, const float *x, QT *w, int S){
if(!coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, &w->xdna,
- w->fmt, w->q4, w->s, w->I, w->O, w->gs, y, x, S))
+ w->fmt, w->q4, w->s, w->I, w->O, w->gs, w->planar, y, x, S))
matmul_qt(y, x, w, S);
}
@@ -219,13 +219,15 @@ int main(int argc, char **argv){
/* ================================================================== */
printf("\npre-device declines -- nothing may reach the device\n");
{
- struct { const char *what; ColiXdnaFamily fam; int fmt,I,O,gs,S; ColiXdnaHard want; } c[] = {
- { "wrong semantic family", COLI_XDNA_FAMILY_NONE, 4,TK,TN,64,TM, COLI_XDNA_HARD_FAMILY_UNSUPPORTED },
- { "sh_down orientation", COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, 4,TN,TK,64,TM, COLI_XDNA_HARD_SHAPE_UNSUPPORTED },
- { "gs != 64", COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, 4,TK,TN,32,TM, COLI_XDNA_HARD_GROUP_SIZE_UNSUPPORTED },
- { "fmt != 4", COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, 2,TK,TN,64,TM, COLI_XDNA_HARD_FORMAT_UNSUPPORTED },
- { "M above range", COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, 4,TK,TN,64,TM+1, COLI_XDNA_HARD_M_OUT_OF_RANGE },
- { "K/N mismatch", COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, 4,TK+8,TN,64,TM, COLI_XDNA_HARD_SHAPE_UNSUPPORTED }
+ struct { const char *what; ColiXdnaFamily fam; int fmt,I,O,gs,planar,S; ColiXdnaHard want; } c[] = {
+ { "wrong semantic family", COLI_XDNA_FAMILY_NONE, 4,TK,TN,64,0,TM, COLI_XDNA_HARD_FAMILY_UNSUPPORTED },
+ { "sh_down orientation", COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, 4,TN,TK,64,0,TM, COLI_XDNA_HARD_SHAPE_UNSUPPORTED },
+ { "gs != 64", COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, 4,TK,TN,32,0,TM, COLI_XDNA_HARD_GROUP_SIZE_UNSUPPORTED },
+ { "fmt != 4", COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, 2,TK,TN,64,0,TM, COLI_XDNA_HARD_FORMAT_UNSUPPORTED },
+ { "M above range", COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, 4,TK,TN,64,0,TM+1, COLI_XDNA_HARD_M_OUT_OF_RANGE },
+ { "K/N mismatch", COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, 4,TK+8,TN,64,0,TM, COLI_XDNA_HARD_SHAPE_UNSUPPORTED },
+ /* K1 planar layout: the converter was never qualified against it */
+ { "planar fmt4 layout", COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, 4,TK,TN,64,1,TM, COLI_XDNA_HARD_LAYOUT_UNSUPPORTED }
};
for(size_t i = 0; i < sizeof c/sizeof c[0]; i++){
use_helper(g_helper);
@@ -239,7 +241,7 @@ int main(int argc, char **argv){
T.fmt = 4; matmul_qt(rr, xx, &T, c[i].S); T.fmt = c[i].fmt;
poison_buf(yy,(size_t)c[i].S*c[i].O);
int handled = coli_xdna_try_matmul(c[i].fam, &T.xdna, c[i].fmt, T.q4, T.s,
- c[i].I, c[i].O, c[i].gs, yy, xx, c[i].S);
+ c[i].I, c[i].O, c[i].gs, c[i].planar, yy, xx, c[i].S);
char m[200];
snprintf(m,sizeof m,"%s -> %s", c[i].what, coli_xdna_hard_text(coli_xdna_test_last_hard()));
ck(coli_xdna_test_last_hard()==c[i].want, m);
@@ -266,7 +268,7 @@ int main(int argc, char **argv){
char none[1200]; snprintf(none,sizeof none,"%s/absent", g_root);
coli_xdna_test_set_artifact_root(none);
float *y = (float*)malloc((size_t)TM*TN*4); poison_buf(y,(size_t)TM*TN);
- coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&W.xdna,4,W.q4,W.s,TK,TN,64,y,x,TM);
+ coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&W.xdna,4,W.q4,W.s,TK,TN,64,0,y,x,TM);
ck(coli_xdna_test_last_hard()==COLI_XDNA_HARD_ARTIFACT_UNAVAILABLE,
"artifact absent -> ARTIFACT_UNAVAILABLE");
ck(coli_xdna_test_helper_calls()==0, "artifact absent -> helper never called");
@@ -274,7 +276,7 @@ int main(int argc, char **argv){
coli_xdna_test_set_artifact_root(g_root);
{ FILE *f = fopen(g_xclbin,"r+b"); if(f){ fseek(f,64,SEEK_SET); fputc(0x5A,f); fclose(f); } }
- coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&W.xdna,4,W.q4,W.s,TK,TN,64,y,x,TM);
+ coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&W.xdna,4,W.q4,W.s,TK,TN,64,0,y,x,TM);
ck(coli_xdna_test_last_hard()==COLI_XDNA_HARD_ARTIFACT_INTEGRITY_FAILED,
"artifact tampered -> ARTIFACT_INTEGRITY_FAILED (not the same as absent)");
ck(coli_xdna_test_helper_calls()==0, "artifact tampered -> unverified bytes never reach the helper");
@@ -282,7 +284,7 @@ int main(int argc, char **argv){
write_blob(g_xclbin, 31u, 4096); build_registry();
g_test_rows[0].correctness_qualified = 0;
- coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&W.xdna,4,W.q4,W.s,TK,TN,64,y,x,TM);
+ coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&W.xdna,4,W.q4,W.s,TK,TN,64,0,y,x,TM);
ck(coli_xdna_test_last_hard()==COLI_XDNA_HARD_ARTIFACT_UNQUALIFIED,
"artifact never correctness-qualified -> ARTIFACT_UNQUALIFIED");
g_test_rows[0].correctness_qualified = 1;
@@ -303,7 +305,7 @@ int main(int argc, char **argv){
coli_xdna_test_set_force_execution(1);
QT T; qt_make(&T, TK, TN, 64, 4242u);
float *y = (float*)malloc((size_t)TM*TN*4); poison_buf(y,(size_t)TM*TN);
- coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&T.xdna,4,T.q4,T.s,TK,TN,64,y,x,TM);
+ coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&T.xdna,4,T.q4,T.s,TK,TN,64,0,y,x,TM);
char m[200];
snprintf(m,sizeof m,"%s -> %s", h[i].what, coli_xdna_hard_text(coli_xdna_test_last_hard()));
ck(coli_xdna_test_last_hard()==h[i].want, m);
@@ -332,7 +334,7 @@ int main(int argc, char **argv){
coli_xdna_test_set_convert_fail_pct(50);
float *y = (float*)malloc((size_t)TM*TN*4); poison_buf(y,(size_t)TM*TN);
int handled = coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&T.xdna,
- 4,T.q4,T.s,TK,TN,64,y,x,TM);
+ 4,T.q4,T.s,TK,TN,64,0,y,x,TM);
ck(handled==0, "mid-conversion failure -> not handled");
ck(coli_xdna_test_last_exec()==COLI_XDNA_EXEC_WEIGHT_PREPARE_FAILED,
"classified WEIGHT_PREPARE_FAILED");
@@ -351,7 +353,7 @@ int main(int argc, char **argv){
coli_xdna_test_set_convert_fail_pct(0);
poison_buf(y,(size_t)TM*TN);
ck(coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&T.xdna,
- 4,T.q4,T.s,TK,TN,64,y,x,TM)==1,
+ 4,T.q4,T.s,TK,TN,64,0,y,x,TM)==1,
"a complete re-preparation recovers and executes");
ck(coli_xdna_prepared_state(T.xdna)==COLI_XDNA_PREP_VALID, "state VALID again");
ck(coli_xdna_test_output_valid()==1, "and the output is valid");
@@ -378,7 +380,7 @@ int main(int argc, char **argv){
float *y = (float*)malloc((size_t)TM*TN*4); poison_buf(y,(size_t)TM*TN);
int d0 = coli_xdna_test_dispatches();
int handled = coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&T.xdna,
- 4,T.q4,T.s,TK,TN,64,y,x,TM);
+ 4,T.q4,T.s,TK,TN,64,0,y,x,TM);
char m[220];
snprintf(m,sizeof m,"%s failure -> not handled", st[i].what); ck(handled==0,m);
snprintf(m,sizeof m,"%s failure -> %s", st[i].what,
@@ -423,7 +425,7 @@ int main(int argc, char **argv){
int d_late = coli_xdna_test_dispatches();
int handled = coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&T.xdna,
- 4,T.q4,T.s,TK,TN,64,y,x,TM);
+ 4,T.q4,T.s,TK,TN,64,0,y,x,TM);
ck(handled==0, "not handled");
ck(coli_xdna_test_last_exec()==COLI_XDNA_EXEC_COMPLETION_FAILED, "COMPLETION_FAILED");
@@ -463,7 +465,7 @@ int main(int argc, char **argv){
QT T; qt_make(&T, TK, TN, 64, 4242u);
float *y = (float*)malloc((size_t)TM*TN*4); poison_buf(y,(size_t)TM*TN);
ColiXdnaExec e = coli_xdna_test_attempt(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&T.xdna,
- 4,T.q4,T.s,TK,TN,64,y,x,TM);
+ 4,T.q4,T.s,TK,TN,64,0,y,x,TM);
ck(e==COLI_XDNA_EXEC_EXECUTE_FAILED, "explicit mode returns the failure class");
ck(coli_xdna_test_output_valid()==0, "output not valid");
ck(count_poison(y,(size_t)TM*TN)==(size_t)TM*TN,
@@ -471,7 +473,7 @@ int main(int argc, char **argv){
arm(F_NONE);
poison_buf(y,(size_t)TM*TN);
e = coli_xdna_test_attempt(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&T.xdna,
- 4,T.q4,T.s,TK,TN,64,y,x,TM);
+ 4,T.q4,T.s,TK,TN,64,0,y,x,TM);
ck(e==COLI_XDNA_EXEC_OK, "explicit mode succeeds when the lane works");
ck(coli_xdna_test_output_valid()==1, "and the output is valid");
ck(count_poison(y,(size_t)TM*TN)==0, "and was written");
@@ -495,11 +497,11 @@ int main(int argc, char **argv){
arm(F_WRAP);
QT T; qt_make(&T, TK, TN, 64, 4242u);
float *y = (float*)malloc((size_t)TM*TN*4);
- coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&T.xdna,4,T.q4,T.s,TK,TN,64,y,x,TM);
+ coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&T.xdna,4,T.q4,T.s,TK,TN,64,0,y,x,TM);
ck(coli_xdna_lane_health()==COLI_XDNA_LANE_HEALTHY,
"a wrap failure leaves the lane HEALTHY -- it is one operation");
arm(F_NONE);
- ck(coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&T.xdna,4,T.q4,T.s,TK,TN,64,y,x,TM)==1,
+ ck(coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&T.xdna,4,T.q4,T.s,TK,TN,64,0,y,x,TM)==1,
"and the very next operation succeeds");
use_helper(g_helper);
@@ -507,12 +509,12 @@ int main(int argc, char **argv){
coli_xdna_test_set_force_execution(1);
arm(F_DEVICE);
QT D; qt_make(&D, TK, TN, 64, 4242u);
- coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&D.xdna,4,D.q4,D.s,TK,TN,64,y,x,TM);
+ coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&D.xdna,4,D.q4,D.s,TK,TN,64,0,y,x,TM);
ck(coli_xdna_lane_health()==COLI_XDNA_LANE_UNAVAILABLE,
"a device-init failure marks the lane UNAVAILABLE for the process");
arm(F_NONE);
int o0 = coli_xdna_test_device_opens();
- coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&D.xdna,4,D.q4,D.s,TK,TN,64,y,x,TM);
+ coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,&D.xdna,4,D.q4,D.s,TK,TN,64,0,y,x,TM);
ck(coli_xdna_test_device_opens()==o0,
"and the device is not re-attempted per operation");
ck(coli_xdna_test_last_hard()==COLI_XDNA_HARD_DEVICE_UNAVAILABLE, "the decline says why");
diff --git a/c/tests/xdna_physical_probe.c b/c/tests/xdna_physical_probe.c
index 62a68318..f1bc0e08 100644
--- a/c/tests/xdna_physical_probe.c
+++ b/c/tests/xdna_physical_probe.c
@@ -188,7 +188,7 @@ int main(int argc, char **argv){
double t0 = now_s();
int handled = coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,
&w->xdna, w->fmt, w->q4, w->s,
- w->I, w->O, w->gs, y, x, S);
+ w->I, w->O, w->gs, w->planar, y, x, S);
double ms = (now_s()-t0)*1000.0;
printf("\n-- %s M=%d --\n", names[t], S);
@@ -278,7 +278,7 @@ int main(int argc, char **argv){
float *y = (float*)malloc((size_t)S*PN*4);
int handled = coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP,
&G.xdna, G.fmt, G.q4, G.s,
- G.I, G.O, G.gs, y, x, S);
+ G.I, G.O, G.gs, G.planar, y, x, S);
printf("\n-- generic f32 activations (BF16 rounding is LOSSY here), M=%d --\n", S);
printf("HANDLED %d\n", handled);
if(!handled) g_bad = 1;
@@ -323,14 +323,14 @@ int main(int argc, char **argv){
float *yb = (float*)malloc((size_t)S*PN*4);
printf("\n-- stale-wrapper control: reprepare a different weight in place --\n");
int ha = coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, &G.xdna,
- G.fmt, G.q4, G.s, G.I, G.O, G.gs, ya, x, S);
+ G.fmt, G.q4, G.s, G.I, G.O, G.gs, G.planar, ya, x, S);
const void *p1 = coli_xdna_prepared_image(G.xdna);
int w0 = coli_xdna_test_userptr_wraps();
coli_xdna_prepared_invalidate(G.xdna);
ColiXdnaPrepResult pr = coli_xdna_prepare_from_fmt4(G.xdna, 4, U.q4, U.s, PK, PN, PGS);
const void *p2 = coli_xdna_prepared_image(G.xdna);
int hb = coli_xdna_try_matmul(COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, &G.xdna,
- G.fmt, G.q4, G.s, G.I, G.O, G.gs, yb, x, S);
+ G.fmt, G.q4, G.s, G.I, G.O, G.gs, G.planar, yb, x, S);
printf("PREPARE_RC %d ptr_same %s\n", (int)pr, p1==p2?"YES":"no");
printf("HANDLED %d %d\n", ha, hb);
printf("WRAPS before=%d after=%d rewrap=%s\n", w0,
@@ -362,7 +362,7 @@ int main(int argc, char **argv){
float *y = (float*)malloc((size_t)8*PN*4);
int before = coli_xdna_test_dispatches();
int h2 = coli_xdna_try_matmul(COLI_XDNA_FAMILY_NONE, &G.xdna, G.fmt, G.q4, G.s,
- G.I, G.O, G.gs, y, x, 8);
+ G.I, G.O, G.gs, G.planar, y, x, 8);
printf("\nWRONG_FAMILY_HANDLED %d dispatch_delta %d (%s)\n", h2,
coli_xdna_test_dispatches()-before,
coli_xdna_hard_text(coli_xdna_test_last_hard()));