Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

inkling: chunked pread for >2GB resident tensors

Nicholas Beerbower32b97a70df51
diff --git a/c/inkling.c b/c/inkling.c
index 87155d97..d56e0ac8 100644
--- a/c/inkling.c
+++ b/c/inkling.c
@@ -382,14 +382,31 @@ static float load_scalar(Model *m, const char *name, float dflt) {
float v; st_read_f32(&m->S, name, &v, 0); return v;
}
+/* chunked pread: a single pread caps at ~2.1 GB on Linux, and the bf16
+ * embed/lm_head tensors are 2.47 GB — loop in 1 GB slices */
+static void pread_all(int fd, void *buf, int64_t nb, int64_t off) {
+ char *p = buf;
+ while (nb > 0) {
+ int64_t chunk = nb < (1<<30) ? nb : (1<<30);
+ ssize_t got = pread(fd, p, (size_t)chunk, off);
+ if (got <= 0) { perror("pread chunk"); exit(1); }
+ p += got; off += got; nb -= got;
+ }
+}
+
/* big matmul weights keep their on-disk dtype resident: BF16 raw (real
* checkpoint, halves RAM), anything else as f32 (tiny oracle: bit-exact) */
static Wt load_w(Model *m, const char *name) {
Wt w = {0};
st_tensor *t = st_find(&m->S, name);
if (!t) { fprintf(stderr, "missing %s\n", name); exit(1); }
- if (t->dtype == 0) { w.h = malloc(t->nbytes); if (!w.h) { fprintf(stderr,"OOM %s\n",name); exit(1); } st_read_raw(&m->S, name, w.h, 0); }
- else { w.f = falloc(t->numel); st_read_f32(&m->S, name, w.f, 0); }
+ if (t->dtype == 0) {
+ w.h = malloc(t->nbytes); if (!w.h) { fprintf(stderr,"OOM %s\n",name); exit(1); }
+ pread_all(t->fd, w.h, t->nbytes, t->off);
+ } else {
+ w.f = falloc(t->numel);
+ st_read_f32(&m->S, name, w.f, 0);
+ }
return w;
}
static Wt wt_off(Wt w, int64_t off) {