Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
/* test_logprob_status.c β the classified logit-row reduction in sample.h and
* the evidence digest in evidence_digest.h.
*
* Both are foundations for engine modes that write artifacts an offline tool
* re-checks, so both need to be right for inputs the sampling path never sees:
* rows containing NaN or an infinity, empty rows, and a target whose distance
* from the row maximum does not fit in a float. A wrong answer there is worse
* than a crash, because it is written into a file and read back later as if it
* were a measurement.
*
* Required properties:
* P1 AGREEMENT β on a row whose logits sit close enough together that a
* float-scale subtraction cannot lose precision, the classified path
* returns exactly the value and argmax flag the plain logprob_target()
* returns. The two paths are not required to agree past a float's own
* precision on a widely spread row: logprob_target() keeps its
* historical float-scale subtraction, while the classified path
* promotes to double first (P1B).
* P1B PRECISION β the classified path's value is accurate to double
* precision: it matches, bit for bit, a reference computed
* a second time in this file by promoting the row's logit to double
* before subtracting, on rows spanning ordinary, widely spread,
* subnormal and full-double-precision inputs. This is the property
* P1's exact test rows are too narrow to exercise.
* P2 CLASSIFICATION β each exceptional row shape reports its own cause, not a
* generic failure, no partition function is invented for it, and the
* precedence between the causes is the documented one.
* P3 PROPAGATION β reading a target out of a row that did not reduce cleanly
* reports the row's original cause, a caller mistake is reported as such
* rather than as a usable value, and every refusal leaves a defined
* number behind rather than whatever the caller's variable held.
* P4 DIGEST β the digest matches published SHA-256 values, including the
* lengths that exercise the block boundary and both padding branches, and
* the streaming form agrees with the one-shot form.
* Exit 0 = all pass.
*/
#include <float.h>
#define main coli_glm_main_unused
#include "../colibri.c"
#undef main
#include "../evidence_digest.h"
static int fails = 0;
#define CHECK(cond,msg) do{ if(!(cond)){ printf(" FAIL: %s\n", msg); fails++; } \
else printf(" ok: %s\n", msg); }while(0)
static int close_enough(double a, double b){
double d = a - b;
if(d < 0) d = -d;
return d <= 1e-12;
}
/* Tight tolerance for P1B: the classified path's value against an
* second double computation should agree to a few ulp; the relative bound leaves no room
* for a float-scale rounding to sneak back in while tolerating, at most,
* a difference in the last bit of a double reduction. */
static int close_enough_precise(double a, double b){
/* a few units in the last place, relative to the reference magnitude */
double tol = 16.0 * DBL_EPSILON * fabs(b) + 1e-300;
return fabs(a - b) <= tol;
}
/* P1: on a row a float subtraction cannot mis-round, the classified path
* agrees with the plain path exactly. */
static void t_agreement(void){
printf("P1 agreement with the plain path\n");
static const float rows[3][5] = {
{ 0.0f, 0.0f, 0.0f, 0.0f, 0.0f },
{ 1.0f, 2.0f, 3.0f, -1.0f, 0.5f },
{ -30.0f, -31.0f, -29.5f, -100.0f, -29.75f },
};
for(int r=0;r<3;r++){
LogprobRow row;
CHECK(logprob_row_checked(rows[r],5,&row)==LOGPROB_FINITE,
"finite row reduces to FINITE");
for(int target=0;target<5;target++){
double classified = 0;
CHECK(logprob_from_row_checked(rows[r],target,&row,&classified)==LOGPROB_FINITE,
"finite row yields a finite target value");
int plain_argmax = 0;
double plain = logprob_target(rows[r],5,target,&plain_argmax);
CHECK(close_enough(classified,plain),
"classified value equals the plain logprob_target value");
CHECK(plain_argmax==(row.argmax==target),
"classified argmax flag equals the plain argmax flag");
}
}
/* A two-entry uniform row has a hand-checkable answer: log(1/2). */
static const float uniform[2] = { 0.0f, 0.0f };
LogprobRow row;
logprob_row_checked(uniform,2,&row);
double value = 0;
logprob_from_row_checked(uniform,0,&row,&value);
CHECK(close_enough(value,-log(2.0)), "uniform pair gives log(1/2)");
CHECK(close_enough(row.logZ,log(2.0)), "uniform pair has logZ = log(2)");
CHECK(row.argmax==0, "uniform pair takes the first maximum");
}
/* A second double computation for P1B (same formula as the code under test, so it
* detects float-first rounding, not ulp-level error): re-finds the row's maximum and
* log-sum-exp from scratch, entirely in double, without touching the
* LogprobRow the function under test already reduced. This is not the same
* code path as logprob_from_row_checked() -- it recomputes the row instead
* of reading r->max/r->logZ -- so it cannot agree with a wrong answer by
* sharing a mistake. */
static double reference_double(const float *lo, int V, int target){
double mx = (double)lo[0];
for(int i=1;i<V;i++){ double v=(double)lo[i]; if(v>mx) mx=v; }
double se = 0;
for(int i=0;i<V;i++) se += exp((double)lo[i]-mx);
return (double)lo[target] - (mx + log(se));
}
static void check_row_precise(const char *name, const float *lo, int V,
const int *targets, int ntargets){
LogprobRow row;
CHECK(logprob_row_checked(lo,V,&row)==LOGPROB_FINITE, name);
for(int k=0;k<ntargets;k++){
int target = targets[k];
double value = 0;
CHECK(logprob_from_row_checked(lo,target,&row,&value)==LOGPROB_FINITE,
"the target reads back finite");
double ref = reference_double(lo,V,target);
CHECK(close_enough_precise(value,ref),
"the classified path's value agrees with the second double computation to a few ulp");
}
}
/* P1B: the classified path agrees with a second double computation (to a few units in
* the last place; ~13 significant digits on the widest rows) on rows P1's own
* exact test rows are too narrow to tell apart from a float-rounded answer.
* Table covers: an ordinary tightly-spread row; a wide-spread row mirroring
* test_ablate_mode.c's t_nll_pin (max 1.0e7, one target's contribution under
* half a float ulp at that magnitude); a moderately spread row whose correct
* value needs digits past a float's own seven to state; and a subnormal row,
* where the magnitudes are so small a float-first subtraction cannot lose
* anything, so this also checks the fix introduces no new problem there. */
static void t_precision(void){
printf("P1B classified path agrees with a second double computation to a few ulp\n");
static const float ordinary[4] = { 2.0f, -1.5f, 0.25f, -3.0f };
int t_ordinary[4] = { 0, 1, 2, 3 };
check_row_precise("ordinary row reduces to FINITE", ordinary, 4, t_ordinary, 4);
/* Mirrors test_ablate_mode.c's t_nll_pin: max 1.0e7 at index 0, the
* target of interest at index 2 with a logit of 0.25 -- a contribution
* the float subtraction lo[2]-max rounds away entirely, since the ulp
* near 1.0e7 is 1.0 and 0.25 is under half of it. */
static float wide_spread[64];
for(int i=0;i<64;i++) wide_spread[i] = -1.0e7f;
wide_spread[0] = 1.0e7f;
wide_spread[2] = 0.25f;
int t_wide[5] = { 0, 1, 2, 5, 63 };
check_row_precise("wide-spread row reduces to FINITE", wide_spread, 64, t_wide, 5);
/* A moderately spread row: max in the tens of thousands, targets with
* fractional digits a float at that magnitude cannot all hold. */
static const float moderate[5] =
{ 10000.0f, 0.123456789f, -5000.5f, 9999.99f, 3.14159265f };
int t_moderate[5] = { 0, 1, 2, 3, 4 };
check_row_precise("moderate-spread row reduces to FINITE", moderate, 5, t_moderate, 5);
/* Subnormal logits: values this small carry no ulp large enough for a
* float-first subtraction to lose, so this is a robustness check as much
* as a precision one. */
static const float subnormal[3] =
{ 1e-45f, 1e-45f*3.0f, 1e-45f*7.0f };
int t_subnormal[3] = { 0, 1, 2 };
check_row_precise("subnormal row reduces to FINITE", subnormal, 3, t_subnormal, 3);
}
/* P2: every exceptional row shape names its own cause. */
static void t_classification(void){
printf("P2 exceptional rows are classified\n");
LogprobRow row;
float nan_row[3] = { 1.0f, NAN, 3.0f };
CHECK(logprob_row_checked(nan_row,3,&row)==LOGPROB_NAN, "NaN present -> NAN");
CHECK(row.status==LOGPROB_NAN, "the returned row carries the same status");
CHECK(row.logZ==0 && row.max==0, "no partition function is invented");
float pos_row[3] = { 1.0f, INFINITY, 3.0f };
CHECK(logprob_row_checked(pos_row,3,&row)==LOGPROB_POS_INF, "+inf present -> POS_INF");
float neg_row[3] = { 1.0f, -INFINITY, 3.0f };
CHECK(logprob_row_checked(neg_row,3,&row)==LOGPROB_NEG_INF, "-inf present -> NEG_INF");
/* Precedence, stated as a fixed order rather than the order of appearance.
* Each row below contains more than one exceptional class, so a change to
* the ranking changes an answer here. */
float mixed_row[3] = { NAN, INFINITY, -INFINITY };
CHECK(logprob_row_checked(mixed_row,3,&row)==LOGPROB_ALL_NONFINITE,
"no finite entry at all -> ALL_NONFINITE, whatever the classes are");
float finite_nan_pos[3] = { 1.0f, NAN, INFINITY };
CHECK(logprob_row_checked(finite_nan_pos,3,&row)==LOGPROB_NAN,
"a NaN outranks a positive infinity");
float finite_pos_nan[3] = { 1.0f, INFINITY, NAN };
CHECK(logprob_row_checked(finite_pos_nan,3,&row)==LOGPROB_NAN,
"a NaN outranks a positive infinity that appears before it");
float finite_nan_neg[3] = { 1.0f, NAN, -INFINITY };
CHECK(logprob_row_checked(finite_nan_neg,3,&row)==LOGPROB_NAN,
"a NaN outranks a negative infinity");
float finite_pos_neg[3] = { 1.0f, INFINITY, -INFINITY };
CHECK(logprob_row_checked(finite_pos_neg,3,&row)==LOGPROB_POS_INF,
"a positive infinity outranks a negative one");
float finite_neg_pos[3] = { 1.0f, -INFINITY, INFINITY };
CHECK(logprob_row_checked(finite_neg_pos,3,&row)==LOGPROB_POS_INF,
"a positive infinity outranks a negative one that appears first");
float finite_neg[3] = { 1.0f, 2.0f, -INFINITY };
CHECK(logprob_row_checked(finite_neg,3,&row)==LOGPROB_NEG_INF,
"a negative infinity alongside finite entries -> NEG_INF");
float all_nan[2] = { NAN, NAN };
CHECK(logprob_row_checked(all_nan,2,&row)==LOGPROB_ALL_NONFINITE,
"a row of NaN alone is still ALL_NONFINITE");
CHECK(logprob_row_checked(NULL,3,&row)==LOGPROB_INVALID, "no row -> INVALID");
CHECK(logprob_row_checked(nan_row,0,&row)==LOGPROB_INVALID, "empty row -> INVALID");
CHECK(logprob_row_checked(nan_row,-1,&row)==LOGPROB_INVALID, "negative length -> INVALID");
CHECK(logprob_row_checked(nan_row,3,NULL)==LOGPROB_NAN,
"the status is returned even with no output row");
CHECK(strcmp(logprob_status_name(LOGPROB_FINITE),"FINITE")==0, "FINITE names itself");
CHECK(strcmp(logprob_status_name(LOGPROB_NAN),"NAN")==0, "NAN names itself");
CHECK(strcmp(logprob_status_name(LOGPROB_POS_INF),"POS_INF")==0, "POS_INF names itself");
CHECK(strcmp(logprob_status_name(LOGPROB_NEG_INF),"NEG_INF")==0, "NEG_INF names itself");
CHECK(strcmp(logprob_status_name(LOGPROB_ALL_NONFINITE),"ALL_NONFINITE")==0,
"ALL_NONFINITE names itself");
CHECK(strcmp(logprob_status_name(LOGPROB_FINITE_OVERFLOW),"FINITE_OVERFLOW")==0,
"FINITE_OVERFLOW names itself");
CHECK(strcmp(logprob_status_name(LOGPROB_INVALID),"INVALID")==0, "INVALID names itself");
}
/* P3: a target read out of a bad row reports the row's cause. A spread that
* would overflow a float subtraction no longer refuses the target, because
* the classified path takes that subtraction in double: it is pinned below
* as a status change, not silently absorbed into "still finite". */
static void t_propagation(void){
printf("P3 target reads propagate the row's cause\n");
LogprobRow row;
float nan_row[3] = { 1.0f, NAN, 3.0f };
logprob_row_checked(nan_row,3,&row);
double value = 12345.0;
CHECK(logprob_from_row_checked(nan_row,0,&row,&value)==LOGPROB_NAN,
"a NaN row propagates NAN to the target read");
CHECK(isnan(value), "a refused target read leaves a defined value behind");
CHECK(logprob_from_row_checked(nan_row,0,NULL,&value)==LOGPROB_INVALID,
"no row -> INVALID");
/* A caller mistake on a perfectly good row must not read back as a usable
* value: the earlier form returned FINITE here and never wrote the output,
* so the caller consumed whatever its own variable happened to hold. */
static const float good_row[3] = { 1.0f, 2.0f, 3.0f };
LogprobRow good;
CHECK(logprob_row_checked(good_row,3,&good)==LOGPROB_FINITE, "the good row reduces");
value = 12345.0;
CHECK(logprob_from_row_checked(NULL,0,&good,&value)==LOGPROB_INVALID,
"no logit vector on a good row -> INVALID, never FINITE");
CHECK(isnan(value), "no logit vector leaves a defined value behind");
value = 12345.0;
CHECK(logprob_from_row_checked(good_row,-1,&good,&value)==LOGPROB_INVALID,
"a negative target on a good row -> INVALID, never FINITE");
CHECK(isnan(value), "a negative target leaves a defined value behind");
logprob_row_checked(nan_row,3,&row);
CHECK(logprob_from_row_checked(NULL,0,&row,&value)==LOGPROB_INVALID,
"no logit vector is a caller mistake even when the row is bad");
/* The widest representable spread: a float subtraction between these two
* values would saturate to infinity even though every logit in the row
* is finite. The classified path subtracts in double, so it does not:
* this is a real, disclosed widening of what the layer accepts, pinned
* here so it cannot regress silently in either direction. */
float wide[2] = { FLT_MAX, -FLT_MAX };
CHECK(logprob_row_checked(wide,2,&row)==LOGPROB_FINITE, "an extreme finite row still reduces");
CHECK(logprob_from_row_checked(wide,1,&row,&value)==LOGPROB_FINITE,
"a target the float subtraction could not hold -> FINITE, not refused, under the double subtraction");
CHECK(value==-2.0*(double)FLT_MAX,
"the saturated target's value is the exact double difference, not an infinity");
CHECK(logprob_from_row_checked(wide,0,&row,&value)==LOGPROB_FINITE,
"the maximum itself still reads back finite");
CHECK(close_enough(value,0.0), "the maximum of a saturated row has log-probability 0");
logprob_row_checked(nan_row,0,&row);
CHECK(logprob_from_row_checked(nan_row,-1,&row,&value)==LOGPROB_INVALID,
"a negative target on an invalid row -> INVALID");
}
/* P4: published SHA-256 answers, chosen to cover the block boundary and both
* padding branches (a message whose tail leaves no room for the length field
* needs an extra block). */
static void t_digest(void){
printf("P4 evidence digest matches published SHA-256 values\n");
static const struct { const char *text; const char *hex; } vectors[] = {
{ "", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" },
{ "abc", "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" },
{ "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq",
"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1" },
{ "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmno",
"2ff100b36c386c65a1afc462ad53e25479bec9498ed00aa5a04de584bc25301b" },
};
char out[65];
for(unsigned v=0;v<sizeof(vectors)/sizeof(vectors[0]);v++){
evidence_sha256_hex(vectors[v].text,strlen(vectors[v].text),out);
CHECK(strcmp(out,vectors[v].hex)==0, "one-shot digest matches the published value");
CHECK(strlen(out)==64, "the digest is 64 characters and NUL-terminated");
}
/* 65 bytes: one full block plus a remainder, fed one byte at a time, so a
* mishandled carry between updates would show up here and not above. */
char sixty_five[66];
memset(sixty_five,'a',65); sixty_five[65]=0;
EvidenceSha256 stream;
unsigned char raw[32];
evidence_sha256_init(&stream);
for(int i=0;i<65;i++) evidence_sha256_update(&stream,sixty_five+i,1);
evidence_sha256_final(&stream,raw);
char streamed[65];
static const char hex[]="0123456789abcdef";
for(int i=0;i<32;i++){ streamed[2*i]=hex[raw[i]>>4]; streamed[2*i+1]=hex[raw[i]&15]; }
streamed[64]=0;
CHECK(strcmp(streamed,
"635361c48bb9eab14198e76ea8ab7f1a41685d6ad62aa9146d301d4f17eb0ae0")==0,
"a byte-at-a-time stream matches the published value");
evidence_sha256_hex(sixty_five,65,out);
CHECK(strcmp(streamed,out)==0, "the streaming and one-shot forms agree");
}
int main(void){
printf("test_logprob_status\n");
t_agreement();
t_precision();
t_classification();
t_propagation();
t_digest();
printf(fails ? "FAILED (%d)\n" : "PASSED (%d failures)\n", fails);
return fails ? 1 : 0;
}