Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(windows): prefetch expert files without blocking inference

JustVugg620f132f3ece
diff --git a/c/Makefile b/c/Makefile
index 42442964..2201d591 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -2115,6 +2115,9 @@ tests/test_i4_acc512$(EXE): tests/test_i4_acc512.c
tests/test_compat_direct$(EXE): tests/test_compat_direct.c
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+tests/test_compat_prefetch$(EXE): tests/test_compat_prefetch.c
+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+
tests/test_expert_store_ops$(EXE): tests/test_expert_store_ops.c
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
diff --git a/c/compat.h b/c/compat.h
index e99cfd81..3b6d2b00 100644
--- a/c/compat.h
+++ b/c/compat.h
@@ -71,7 +71,7 @@ static inline int compat_open_direct(const char *path){
* text-mode translation β€” NEVER use
* _read/_lseeki64 which are racy AND
* corrupt 0x0A bytes in binary files).
- * posix_fadvise -> no-op (advisory only; macOS already no-ops DONTNEED).
+ * posix_fadvise -> bounded background WILLNEED; other advice is a no-op.
* mlock -> compat_mlock (VirtualLock + crescita working set).
* posix_memalign->_aligned_malloc(free must be compat_aligned_free).
* rename -> compat_rename (MoveFileEx MOVEFILE_REPLACE_EXISTING;
@@ -112,18 +112,11 @@ static inline int compat_open_direct(const char *path){
#define COMPAT_O_RDONLY (O_RDONLY | O_BINARY)
#define COMPAT_O_BINARY O_BINARY
-/* --- posix_fadvise: Windows has no direct equivalent. Semantics:
- * WILLNEED -> warm the OS page cache so a later synchronous pread finds the
- * pages resident. Implemented as an overlapped background ReadFile
- * into a throwaway scratch buffer (fire-and-forget readahead). Called
- * from the dedicated PILOT I/O thread / next-block readahead in moe(),
- * NEVER inline on the hot path (the existing comment at glm.c:2847
- * measures inline fadvise submit at ~0.5ms x 169k calls = +92s/48tok).
- * Each call owns its OVERLAPPED + scratch buffer -> thread-safe.
- * DONTNEED -> no-op: Windows' standby-list trimming self-regulates under pressure,
- * and on a low-RAM host keeping the pages is what we want for reuse.
- * Matches macOS (compat.h:16-19) which no-ops DONTNEED for the same
- * reason. The engine only ever uses DONTNEED as an advisory. */
+/* WILLNEED schedules bounded background prefetch, including for ordinary
+ * synchronous CRT file handles. Supplying OVERLAPPED to ReadFile alone does
+ * not make such handles asynchronous. The worker owns a duplicate handle and
+ * prefetches a read-only mapping without a throwaway copy of the expert.
+ * DONTNEED remains advisory/no-op: Windows manages standby pages itself. */
#ifndef POSIX_FADV_NORMAL
#define POSIX_FADV_NORMAL 0
#define POSIX_FADV_RANDOM 1
@@ -132,27 +125,9 @@ static inline int compat_open_direct(const char *path){
#define POSIX_FADV_DONTNEED 4
#define POSIX_FADV_NOREUSE 5
#endif
+#include "compat_prefetch_win.h"
static inline int compat_fadvise(int fd, off_t off, off_t len, int advice){
- if(advice!=POSIX_FADV_WILLNEED || len<=0) return 0;
- intptr_t osfh=_get_osfhandle(fd);
- if(osfh==-1 || osfh==-2) return 0;
- HANDLE h=(HANDLE)osfh;
- /* Cap the readahead window: reading a whole 19MB expert per hint is fine on the
- * PILOT thread, but a pathological huge len would spike transient memory. */
- size_t rdlen = (len>(off_t)(64*1024*1024)) ? (size_t)(64*1024*1024) : (size_t)len;
- char *buf=(char*)_aligned_malloc(rdlen, 4096);
- if(!buf) return -1;
- OVERLAPPED ov={0};
- ov.Offset = (DWORD)( (off_t)off & 0xFFFFFFFFULL);
- ov.OffsetHigh = (DWORD)(((off_t)off >> 32) & 0xFFFFFFFFULL);
- /* Issue an overlapped read. With a non-OVERLAPPED-opened handle ReadFile still
- * accepts lpOverlapped (it carries the 64-bit offset) and blocks until the read
- * completes β€” but crucially it populates the standby page cache for this region,
- * so the later synchronous pread on the same offsets faults from RAM not disk. */
- DWORD got=0;
- ReadFile(h, buf, (DWORD)rdlen, &got, &ov);
- _aligned_free(buf);
- return 0;
+ return advice == POSIX_FADV_WILLNEED ? compat_prefetch_file(fd, off, len) : 0;
}
#define posix_fadvise compat_fadvise
diff --git a/c/compat_prefetch_win.h b/c/compat_prefetch_win.h
new file mode 100644
index 00000000..af2af737
--- /dev/null
+++ b/c/compat_prefetch_win.h
@@ -0,0 +1,88 @@
+/* Bounded, advisory file readahead for Windows. Included by compat.h after
+ * windows.h. No model pointers or CRT descriptors escape into a worker. */
+#ifndef COLI_COMPAT_PREFETCH_WIN_H
+#define COLI_COMPAT_PREFETCH_WIN_H
+
+#define COMPAT_PREFETCH_JOBS 4
+#define COMPAT_PREFETCH_BYTES (64u * 1024u * 1024u)
+
+/* Keep the Windows 7 build target usable: the hint is optional on systems
+ * without PrefetchVirtualMemory. This is its Windows 8 range ABI. */
+typedef struct { PVOID VirtualAddress; SIZE_T NumberOfBytes; } CompatPrefetchRange;
+typedef BOOL (WINAPI *CompatPrefetchFn)(HANDLE, ULONG_PTR, CompatPrefetchRange *, ULONG);
+typedef struct {
+ HANDLE file;
+ HMODULE module;
+ unsigned long long offset;
+ SIZE_T bytes;
+} CompatPrefetchJob;
+static volatile LONG compat_prefetch_pending;
+
+static VOID CALLBACK compat_prefetch_run(PTP_CALLBACK_INSTANCE instance, PVOID context) {
+ CompatPrefetchJob *job = (CompatPrefetchJob *)context;
+ /* An SDK DLL may be released as soon as its caller returns. The worker
+ * owns a module reference until the thread pool has left this callback. */
+ if (job->module) FreeLibraryWhenCallbackReturns(instance, job->module);
+ CallbackMayRunLong(instance);
+ CompatPrefetchFn prefetch = (CompatPrefetchFn)(void *)GetProcAddress(
+ GetModuleHandleA("kernel32.dll"), "PrefetchVirtualMemory");
+ LARGE_INTEGER size;
+ if (prefetch && GetFileSizeEx(job->file, &size) && size.QuadPart > 0 &&
+ job->offset < (unsigned long long)size.QuadPart) {
+ unsigned long long left = (unsigned long long)size.QuadPart - job->offset;
+ SIZE_T bytes = left < job->bytes ? (SIZE_T)left : job->bytes;
+ SYSTEM_INFO info; GetSystemInfo(&info);
+ unsigned long long base = job->offset - job->offset % info.dwAllocationGranularity;
+ SIZE_T delta = (SIZE_T)(job->offset - base);
+ HANDLE mapping = CreateFileMappingW(job->file, NULL, PAGE_READONLY, 0, 0, NULL);
+ if (mapping) {
+ void *view = MapViewOfFile(mapping, FILE_MAP_READ, (DWORD)(base >> 32),
+ (DWORD)base, delta + bytes);
+ if (view) {
+ CompatPrefetchRange range = {(char *)view + delta, bytes};
+ prefetch(GetCurrentProcess(), 1, &range, 0);
+ UnmapViewOfFile(view);
+ }
+ CloseHandle(mapping);
+ }
+ }
+ CloseHandle(job->file);
+ free(job);
+ InterlockedDecrement(&compat_prefetch_pending);
+}
+
+static inline int compat_prefetch_file(int fd, off_t off, off_t len) {
+ if (fd < 0 || off < 0 || len <= 0) return 0;
+ /* Drop excess hints instead of queueing stale experts or blocking inference.
+ * The worker maps at most 64 MiB and allocates no read-sized scratch copy. */
+ LONG n = InterlockedCompareExchange(&compat_prefetch_pending, 0, 0);
+ for (;;) {
+ if (n >= COMPAT_PREFETCH_JOBS) return 0;
+ LONG was = InterlockedCompareExchange(&compat_prefetch_pending, n + 1, n);
+ if (was == n) break;
+ n = was;
+ }
+ intptr_t osfh = _get_osfhandle(fd);
+ HANDLE file = NULL;
+ HMODULE module = NULL;
+ CompatPrefetchJob *job = NULL;
+ if (osfh == -1 || osfh == -2 ||
+ !DuplicateHandle(GetCurrentProcess(), (HANDLE)osfh, GetCurrentProcess(),
+ &file, 0, FALSE, DUPLICATE_SAME_ACCESS)) goto fail;
+ if (!GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS,
+ (LPCSTR)(void *)compat_prefetch_run, &module)) goto fail;
+ /* Executables cannot be unloaded; only a DLL needs deferred release. */
+ if (module == GetModuleHandleA(NULL)) module = NULL;
+ job = (CompatPrefetchJob *)malloc(sizeof(*job));
+ if (!job) goto fail;
+ *job = (CompatPrefetchJob){file, module, (unsigned long long)off,
+ len > (off_t)COMPAT_PREFETCH_BYTES ? COMPAT_PREFETCH_BYTES : (SIZE_T)len};
+ if (TrySubmitThreadpoolCallback(compat_prefetch_run, job, NULL)) return 0;
+fail:
+ if (file) CloseHandle(file);
+ if (module) FreeLibrary(module);
+ free(job);
+ InterlockedDecrement(&compat_prefetch_pending);
+ return 0; /* advice is never required for correctness */
+}
+#endif
diff --git a/c/tests/test_compat_direct.c b/c/tests/test_compat_direct.c
index ca9c7b45..897f186b 100644
--- a/c/tests/test_compat_direct.c
+++ b/c/tests/test_compat_direct.c
@@ -51,8 +51,8 @@ int main(void){
if(compat_open_direct("no_such_file.tmp")>=0) return fail("open missing file must fail");
if(compat_fsize(-1)>=0) return fail("compat_fsize on bad fd must be negative");
- /* compat_fadvise: WILLNEED warms the page cache (background read into throwaway
- * buffer), DONTNEED is a documented no-op. After a WILLNEED the buffered fd's
+ /* compat_fadvise: WILLNEED schedules bounded background readahead,
+ * DONTNEED is a documented no-op. After a WILLNEED the buffered fd's
* subsequent pread must still return the exact bytes β€” the cache-warmer must not
* corrupt data. Bad fd / non-WILLNEED advice must be safe no-ops (return 0). */
int wfd = open(TMPF, COMPAT_O_RDONLY);
diff --git a/c/tests/test_compat_prefetch.c b/c/tests/test_compat_prefetch.c
new file mode 100644
index 00000000..4c5eb8e2
--- /dev/null
+++ b/c/tests/test_compat_prefetch.c
@@ -0,0 +1,144 @@
+/* Windows readahead must never read weights on the caller, grow an unbounded
+ * queue, or follow a recycled CRT descriptor. Real mappings and thread-pool
+ * workers are used; a gate makes scheduling and resource failures deterministic. */
+#include <stdio.h>
+#ifndef _WIN32
+int main(void) { puts("compat prefetch: skipped (native POSIX advice)"); return 0; }
+#else
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include <windows.h>
+#include <io.h>
+#include <fcntl.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+
+static HANDLE gate;
+static volatile LONG submitted, completed, inspected, reads, bad;
+static int reject_submit, reject_duplicate, missing_api;
+static SIZE_T expected_bytes;
+static unsigned char expected_value;
+typedef struct { PVOID VirtualAddress; SIZE_T NumberOfBytes; } ProbeRange;
+typedef BOOL (WINAPI *PrefetchFn)(HANDLE, ULONG_PTR, ProbeRange *, ULONG);
+
+static BOOL WINAPI probe_prefetch(HANDLE process, ULONG_PTR count, ProbeRange *ranges, ULONG flags) {
+ if (count != 1 || flags || ranges[0].NumberOfBytes != expected_bytes ||
+ *(unsigned char *)ranges[0].VirtualAddress != expected_value)
+ InterlockedIncrement(&bad);
+ InterlockedIncrement(&inspected);
+ PrefetchFn fn = (PrefetchFn)(void *)GetProcAddress(GetModuleHandleA("kernel32.dll"), "PrefetchVirtualMemory");
+ return fn ? fn(process, count, ranges, flags) : TRUE;
+}
+static FARPROC WINAPI probe_get_proc(HMODULE module, LPCSTR name) {
+ if (!strcmp(name, "PrefetchVirtualMemory"))
+ return missing_api ? NULL : (FARPROC)(void *)probe_prefetch;
+ return GetProcAddress(module, name);
+}
+typedef struct { PTP_SIMPLE_CALLBACK fn; PVOID arg; } ProbeWork;
+static VOID CALLBACK probe_worker(PTP_CALLBACK_INSTANCE instance, PVOID arg) {
+ ProbeWork work = *(ProbeWork *)arg; free(arg);
+ if (WaitForSingleObject(gate, 10000) != WAIT_OBJECT_0) InterlockedIncrement(&bad);
+ work.fn(instance, work.arg);
+ InterlockedIncrement(&completed);
+}
+static BOOL WINAPI probe_submit(PTP_SIMPLE_CALLBACK fn, PVOID arg, PTP_CALLBACK_ENVIRON env) {
+ if (reject_submit) return FALSE;
+ ProbeWork *work = malloc(sizeof(*work));
+ if (!work) return FALSE;
+ *work = (ProbeWork){fn, arg};
+ if (!TrySubmitThreadpoolCallback(probe_worker, work, env)) { free(work); return FALSE; }
+ InterlockedIncrement(&submitted);
+ return TRUE;
+}
+static BOOL WINAPI probe_duplicate(HANDLE a, HANDLE b, HANDLE c, LPHANDLE d, DWORD e, BOOL f, DWORD g) {
+ return !reject_duplicate && DuplicateHandle(a, b, c, d, e, f, g);
+}
+static BOOL WINAPI probe_read(HANDLE a, LPVOID b, DWORD c, LPDWORD d, LPOVERLAPPED e) {
+ InterlockedIncrement(&reads);
+ return ReadFile(a, b, c, d, e);
+}
+#define GetProcAddress probe_get_proc
+#define TrySubmitThreadpoolCallback probe_submit
+#define DuplicateHandle probe_duplicate
+#define ReadFile probe_read
+#include "../compat.h"
+#undef GetProcAddress
+#undef TrySubmitThreadpoolCallback
+#undef DuplicateHandle
+#undef ReadFile
+
+#define CHECK(x) do { if (!(x)) { fprintf(stderr, "FAIL line %d: %s\n", __LINE__, #x); return 1; } } while (0)
+static int drain(void) {
+ SetEvent(gate);
+ ULONGLONG end = GetTickCount64() + 10000;
+ while (InterlockedCompareExchange(&completed, 0, 0) != submitted && GetTickCount64() < end) Sleep(1);
+ if (completed != submitted || compat_prefetch_pending || bad)
+ fprintf(stderr, "drain: submitted=%ld completed=%ld pending=%ld inspected=%ld bad=%ld\n",
+ submitted, completed, compat_prefetch_pending, inspected, bad);
+ return completed == submitted && !compat_prefetch_pending && !bad;
+}
+int main(void) {
+ gate = CreateEventW(NULL, TRUE, FALSE, NULL); CHECK(gate);
+ const char *path = "test_prefetch.tmp";
+ int fd = _open(path, _O_RDWR | _O_CREAT | _O_TRUNC | _O_BINARY, _S_IREAD | _S_IWRITE);
+ CHECK(fd >= 0);
+ unsigned char data[4096]; memset(data, 0x5a, sizeof data);
+ CHECK(_write(fd, data, sizeof data) == sizeof data);
+ expected_value = 0x5a; expected_bytes = 4001;
+ for (int i = 0; i < 100; i++) CHECK(!posix_fadvise(fd, 95, 4001, POSIX_FADV_WILLNEED));
+ CHECK(submitted == COMPAT_PREFETCH_JOBS);
+ CHECK(compat_prefetch_pending == COMPAT_PREFETCH_JOBS);
+ CHECK(reads == 0); /* no synchronous pre-read on the caller */
+ CHECK(_lseeki64(fd, 0, SEEK_CUR) == sizeof data);
+ int old = fd; _close(fd);
+ fd = _open("test_prefetch_reused.tmp", _O_RDWR | _O_CREAT | _O_TRUNC | _O_BINARY, _S_IREAD | _S_IWRITE);
+ CHECK(fd == old); /* queued work must retain the original file */
+ memset(data, 0x33, sizeof data); CHECK(_write(fd, data, sizeof data) == sizeof data);
+ CHECK(drain()); CHECK(inspected == COMPAT_PREFETCH_JOBS);
+ CHECK(remove(path) == 0); /* no duplicate handle or mapping left open */
+
+ DWORD handles_before, handles_after;
+ CHECK(GetProcessHandleCount(GetCurrentProcess(), &handles_before));
+ reject_submit = 1;
+ for (int i = 0; i < 100; i++) CHECK(!posix_fadvise(fd, 0, 4096, POSIX_FADV_WILLNEED));
+ reject_submit = 0; reject_duplicate = 1;
+ for (int i = 0; i < 100; i++) CHECK(!posix_fadvise(fd, 0, 4096, POSIX_FADV_WILLNEED));
+ reject_duplicate = 0;
+ CHECK(!compat_prefetch_pending && submitted == COMPAT_PREFETCH_JOBS);
+ CHECK(GetProcessHandleCount(GetCurrentProcess(), &handles_after));
+ CHECK(handles_after <= handles_before);
+
+ /* >4 GiB, unaligned offset and a hint crossing EOF. The sparse file avoids
+ * allocating its hole on disk; only the final page is written. */
+ HANDLE h = (HANDLE)_get_osfhandle(fd);
+ DWORD got;
+ CHECK(DeviceIoControl(h, FSCTL_SET_SPARSE, NULL, 0, NULL, 0, &got, NULL));
+ off_t offset = ((off_t)1 << 32) + 8192 + 19;
+ CHECK(_lseeki64(fd, offset, SEEK_SET) == offset);
+ CHECK(_write(fd, data, sizeof data) == sizeof data);
+ expected_value = 0x33; expected_bytes = sizeof data;
+ CHECK(!posix_fadvise(fd, offset, 65536, POSIX_FADV_WILLNEED)); CHECK(drain());
+ CHECK(inspected == COMPAT_PREFETCH_JOBS + 1);
+
+ /* Capped window, missing API and empty/invalid ranges are only hints. */
+ expected_bytes = COMPAT_PREFETCH_BYTES; expected_value = 0x33;
+ CHECK(!posix_fadvise(fd, 0, offset, POSIX_FADV_WILLNEED)); CHECK(drain());
+ LONG before = inspected;
+ missing_api = 1;
+ CHECK(!posix_fadvise(fd, 0, 4096, POSIX_FADV_WILLNEED)); CHECK(drain());
+ missing_api = 0;
+ CHECK(!posix_fadvise(fd, offset + 65536, 4096, POSIX_FADV_WILLNEED)); CHECK(drain());
+ CHECK(inspected == before);
+ LONG jobs = submitted;
+ CHECK(!posix_fadvise(fd, 0, 4096, POSIX_FADV_DONTNEED));
+ CHECK(!posix_fadvise(fd, -1, 4096, POSIX_FADV_WILLNEED));
+ CHECK(!posix_fadvise(-1, 0, 4096, POSIX_FADV_WILLNEED));
+ CHECK(!posix_fadvise(fd, 0, 0, POSIX_FADV_WILLNEED));
+ CHECK(submitted == jobs && !reads);
+ _close(fd); CHECK(!remove("test_prefetch_reused.tmp"));
+ CloseHandle(gate);
+ puts("compat prefetch: PASS (async, bounded, fd reuse, >4 GiB, EOF, failures)");
+ return 0;
+}
+#endif
diff --git a/docs/windows.md b/docs/windows.md
index ebf9e6c6..2308cd5d 100644
--- a/docs/windows.md
+++ b/docs/windows.md
@@ -1,5 +1,14 @@
# Windows 11 native install β€” a complete walkthrough (no WSL)
+Expert readahead uses a bounded Windows thread-pool queue. A `WILLNEED` hint
+returns without reading the expert on the inference thread, even when the file
+was opened synchronously. The worker prefetches a read-only file mapping;
+there is no second, temporary copy of the expert. At most four hints per engine
+translation unit are pending, with each window capped at 64 MiB. Excess hints,
+unavailable APIs and failed mappings are safely skipped; ordinary model reads
+still supply the weights. This applies to the common file-advice path used by
+the CPU and GPU engines and does not change their weight formats or arithmetic.
+
A start-to-finish, reproducible path from a fresh Windows 11 machine to GLM-5.2 generating tokens, with the GPU tier. Every step and every failure mode below was hit and verified on real hardware: Core Ultra 9 285K (AVX-VNNI) / RTX 5080 (sm_120) / 128 GB RAM / Windows 11 24H2 (issue #306). Steps are ordered so the long downloads run while you build.
---