Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
/* test_glm53_image_key β the KV reuse decision when a request carries an image.
*
* Two requests can have identical token ids and different pictures: the
* gateway expands every image into the same run of `image_token_id`, and the
* pixels arrive in their own frame. If the slot compared ids alone, the second
* request would answer from the attention state built on the first picture β
* no crash, a plausible reply about the wrong photo. So the state-identity
* rule gets its own cases here: same placeholders, different bytes β nothing
* shared; same bytes β everything shared, and the tower outputs the prefix
* already holds are counted so the caller can skip encoding them.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "../kv_image_key.h"
static int failures;
#define CHECK(cond, ...) do { if (!(cond)) { \
fprintf(stderr, "FAIL %s:%d: ", __FILE__, __LINE__); \
fprintf(stderr, __VA_ARGS__); fputc('\n', stderr); failures++; } } while (0)
/* The reuse rule glm53.c applies: the cached prefix is taken whole or not at
* all, and there must be at least one new position to prefill. */
static int reuse(const uint64_t *have, int cached, const uint64_t *want, int n) {
if (cached <= 0 || cached >= n) return 0;
return kv_keys_shared(have, cached, want, n) >= cached ? cached : 0;
}
int main(void) {
enum { IMG = 5, VOCAB = 200000 };
const float a[8] = { 0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f, 0.7f, 0.8f };
float b[8]; memcpy(b, a, sizeof(a)); b[3] = 0.41f; /* one patch value off */
/* ---- the hash names content, not the request ---- */
const uint64_t ha = kv_image_hash(a, sizeof(a), 4, 4);
CHECK(ha == kv_image_hash(a, sizeof(a), 4, 4), "same bytes, same grid: same hash");
CHECK(ha != kv_image_hash(b, sizeof(b), 4, 4), "one value off: another image");
CHECK(ha != kv_image_hash(a, sizeof(a), 2, 8), "same bytes at another grid: another image");
CHECK(ha != kv_image_hash(a, sizeof(a) - 4, 4, 4), "a truncated payload is another image");
/* ---- keys never collide with token ids ---- */
const uint64_t ka = kv_image_key(ha), kb = kv_image_key(kv_image_hash(b, sizeof(b), 4, 4));
CHECK(kv_key_is_image(ka) && kv_key_is_image(kb), "image keys carry the image bit");
CHECK(kv_key_is_image(KV_IMAGE_KEY_NONE), "the frameless placeholder is still an image position");
CHECK(ka != KV_IMAGE_KEY_NONE, "a real image is not the frameless placeholder");
{
const int ids[3] = { 0, VOCAB - 1, 0x7fffffff };
uint64_t keys[3];
kv_keys_build(keys, ids, 3, IMG, ka);
for (int t = 0; t < 3; t++) {
CHECK(!kv_key_is_image(keys[t]), "token id %d must not read as an image", ids[t]);
CHECK(keys[t] == (uint64_t)ids[t], "token id %d keeps its id", ids[t]);
}
const int neg[1] = { -1 };
kv_keys_build(keys, neg, 1, IMG, ka);
CHECK(!kv_key_is_image(keys[0]), "a negative id must not read as an image");
kv_keys_build(keys, ids, 3, -1, ka); /* checkpoint without an image token */
CHECK(keys[0] == 0 && keys[2] == 0x7fffffff, "no image token: nothing is substituted");
}
/* ---- T10: identical placeholders, different content β no shared KV ---- */
const int turn1[] = { 7, IMG, IMG, IMG, 9 }; /* what the slot holds */
const int turn2[] = { 7, IMG, IMG, IMG, 9, 11 }; /* extends it by one token */
const int cached = 5;
uint64_t have[5], want[6];
kv_keys_build(have, turn1, cached, IMG, ka); /* built on picture A */
kv_keys_build(want, turn2, 6, IMG, kb); /* picture B, same ids */
CHECK(kv_keys_shared(have, cached, want, 6) == 1,
"different image: agreement stops at the first placeholder");
CHECK(reuse(have, cached, want, 6) == 0, "different image: no reuse");
kv_keys_build(want, turn2, 6, IMG, ka); /* picture A again */
CHECK(kv_keys_shared(have, cached, want, 6) == cached, "same image: whole prefix agrees");
CHECK(reuse(have, cached, want, 6) == cached, "same image: the prefix is reused");
CHECK(kv_keys_images(want, cached) == 3,
"the reused prefix already holds all 3 tower outputs");
CHECK(kv_keys_images(want, 1) == 0, "no image position before the span");
CHECK(kv_keys_images(want, 2) == 1, "a prefix ending mid-span holds one");
kv_keys_build(want, turn2, 6, IMG, KV_IMAGE_KEY_NONE); /* placeholders, no frame */
CHECK(reuse(have, cached, want, 6) == 0, "no frame: not the picture the slot holds");
/* the slot built on frameless placeholders matches only the same */
kv_keys_build(have, turn1, cached, IMG, KV_IMAGE_KEY_NONE);
CHECK(reuse(have, cached, want, 6) == cached, "frameless then frameless: same input");
kv_keys_build(want, turn2, 6, IMG, ka);
CHECK(reuse(have, cached, want, 6) == 0, "frameless then picture A: not the same input");
/* ---- the text rules did not move ---- */
const int text1[] = { 7, 8, 9 }, text2[] = { 7, 8, 9, 10 }, other[] = { 7, 99, 9, 10 };
uint64_t t1[3], t2[4], t3[4];
kv_keys_build(t1, text1, 3, IMG, ka);
kv_keys_build(t2, text2, 4, IMG, ka);
kv_keys_build(t3, other, 4, IMG, ka);
CHECK(reuse(t1, 3, t2, 4) == 3, "an extending text prompt reuses all 3");
CHECK(reuse(t1, 3, t3, 4) == 0, "a diverging text prompt reuses nothing");
CHECK(reuse(t1, 3, t1, 3) == 0, "equal length: nothing new to prefill, no reuse");
CHECK(reuse(t1, 0, t2, 4) == 0, "an empty slot reuses nothing");
if (failures) { fprintf(stderr, "%d failure(s)\n", failures); return 1; }
puts("test_glm53_image_key: ok");
return 0;
}