Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
fix(windows): prefetch expert files without blocking inference
JustVugg620f132f3ece
diff --git a/c/Makefile b/c/Makefileindex 42442964..2201d591 100644--- a/c/Makefile+++ b/c/Makefile@@ -2115,6 +2115,9 @@ tests/test_i4_acc512$(EXE): tests/test_i4_acc512.ctests/test_compat_direct$(EXE): tests/test_compat_direct.c$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+tests/test_compat_prefetch$(EXE): tests/test_compat_prefetch.c+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+tests/test_expert_store_ops$(EXE): tests/test_expert_store_ops.c$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)diff --git a/c/compat.h b/c/compat.hindex e99cfd81..3b6d2b00 100644--- a/c/compat.h+++ b/c/compat.h@@ -71,7 +71,7 @@ static inline int compat_open_direct(const char *path){* text-mode translation β NEVER use* _read/_lseeki64 which are racy AND* corrupt 0x0A bytes in binary files).- * posix_fadvise -> no-op (advisory only; macOS already no-ops DONTNEED).+ * posix_fadvise -> bounded background WILLNEED; other advice is a no-op.* mlock -> compat_mlock (VirtualLock + crescita working set).* posix_memalign->_aligned_malloc(free must be compat_aligned_free).* rename -> compat_rename (MoveFileEx MOVEFILE_REPLACE_EXISTING;@@ -112,18 +112,11 @@ static inline int compat_open_direct(const char *path){#define COMPAT_O_RDONLY (O_RDONLY | O_BINARY)#define COMPAT_O_BINARY O_BINARY-/* --- posix_fadvise: Windows has no direct equivalent. Semantics:- * WILLNEED -> warm the OS page cache so a later synchronous pread finds the- * pages resident. Implemented as an overlapped background ReadFile- * into a throwaway scratch buffer (fire-and-forget readahead). Called- * from the dedicated PILOT I/O thread / next-block readahead in moe(),- * NEVER inline on the hot path (the existing comment at glm.c:2847- * measures inline fadvise submit at ~0.5ms x 169k calls = +92s/48tok).- * Each call owns its OVERLAPPED + scratch buffer -> thread-safe.- * DONTNEED -> no-op: Windows' standby-list trimming self-regulates under pressure,- * and on a low-RAM host keeping the pages is what we want for reuse.- * Matches macOS (compat.h:16-19) which no-ops DONTNEED for the same- * reason. The engine only ever uses DONTNEED as an advisory. */+/* WILLNEED schedules bounded background prefetch, including for ordinary+ * synchronous CRT file handles. Supplying OVERLAPPED to ReadFile alone does+ * not make such handles asynchronous. The worker owns a duplicate handle and+ * prefetches a read-only mapping without a throwaway copy of the expert.+ * DONTNEED remains advisory/no-op: Windows manages standby pages itself. */#ifndef POSIX_FADV_NORMAL#define POSIX_FADV_NORMAL 0#define POSIX_FADV_RANDOM 1@@ -132,27 +125,9 @@ static inline int compat_open_direct(const char *path){#define POSIX_FADV_DONTNEED 4#define POSIX_FADV_NOREUSE 5#endif+#include "compat_prefetch_win.h"static inline int compat_fadvise(int fd, off_t off, off_t len, int advice){- if(advice!=POSIX_FADV_WILLNEED || len<=0) return 0;- intptr_t osfh=_get_osfhandle(fd);- if(osfh==-1 || osfh==-2) return 0;- HANDLE h=(HANDLE)osfh;- /* Cap the readahead window: reading a whole 19MB expert per hint is fine on the- * PILOT thread, but a pathological huge len would spike transient memory. */- size_t rdlen = (len>(off_t)(64*1024*1024)) ? (size_t)(64*1024*1024) : (size_t)len;- char *buf=(char*)_aligned_malloc(rdlen, 4096);- if(!buf) return -1;- OVERLAPPED ov={0};- ov.Offset = (DWORD)( (off_t)off & 0xFFFFFFFFULL);- ov.OffsetHigh = (DWORD)(((off_t)off >> 32) & 0xFFFFFFFFULL);- /* Issue an overlapped read. With a non-OVERLAPPED-opened handle ReadFile still- * accepts lpOverlapped (it carries the 64-bit offset) and blocks until the read- * completes β but crucially it populates the standby page cache for this region,- * so the later synchronous pread on the same offsets faults from RAM not disk. */- DWORD got=0;- ReadFile(h, buf, (DWORD)rdlen, &got, &ov);- _aligned_free(buf);- return 0;+ return advice == POSIX_FADV_WILLNEED ? compat_prefetch_file(fd, off, len) : 0;}#define posix_fadvise compat_fadvisediff --git a/c/compat_prefetch_win.h b/c/compat_prefetch_win.hnew file mode 100644index 00000000..af2af737--- /dev/null+++ b/c/compat_prefetch_win.h@@ -0,0 +1,88 @@+/* Bounded, advisory file readahead for Windows. Included by compat.h after+ * windows.h. No model pointers or CRT descriptors escape into a worker. */+#ifndef COLI_COMPAT_PREFETCH_WIN_H+#define COLI_COMPAT_PREFETCH_WIN_H++#define COMPAT_PREFETCH_JOBS 4+#define COMPAT_PREFETCH_BYTES (64u * 1024u * 1024u)++/* Keep the Windows 7 build target usable: the hint is optional on systems+ * without PrefetchVirtualMemory. This is its Windows 8 range ABI. */+typedef struct { PVOID VirtualAddress; SIZE_T NumberOfBytes; } CompatPrefetchRange;+typedef BOOL (WINAPI *CompatPrefetchFn)(HANDLE, ULONG_PTR, CompatPrefetchRange *, ULONG);+typedef struct {+ HANDLE file;+ HMODULE module;+ unsigned long long offset;+ SIZE_T bytes;+} CompatPrefetchJob;+static volatile LONG compat_prefetch_pending;++static VOID CALLBACK compat_prefetch_run(PTP_CALLBACK_INSTANCE instance, PVOID context) {+ CompatPrefetchJob *job = (CompatPrefetchJob *)context;+ /* An SDK DLL may be released as soon as its caller returns. The worker+ * owns a module reference until the thread pool has left this callback. */+ if (job->module) FreeLibraryWhenCallbackReturns(instance, job->module);+ CallbackMayRunLong(instance);+ CompatPrefetchFn prefetch = (CompatPrefetchFn)(void *)GetProcAddress(+ GetModuleHandleA("kernel32.dll"), "PrefetchVirtualMemory");+ LARGE_INTEGER size;+ if (prefetch && GetFileSizeEx(job->file, &size) && size.QuadPart > 0 &&+ job->offset < (unsigned long long)size.QuadPart) {+ unsigned long long left = (unsigned long long)size.QuadPart - job->offset;+ SIZE_T bytes = left < job->bytes ? (SIZE_T)left : job->bytes;+ SYSTEM_INFO info; GetSystemInfo(&info);+ unsigned long long base = job->offset - job->offset % info.dwAllocationGranularity;+ SIZE_T delta = (SIZE_T)(job->offset - base);+ HANDLE mapping = CreateFileMappingW(job->file, NULL, PAGE_READONLY, 0, 0, NULL);+ if (mapping) {+ void *view = MapViewOfFile(mapping, FILE_MAP_READ, (DWORD)(base >> 32),+ (DWORD)base, delta + bytes);+ if (view) {+ CompatPrefetchRange range = {(char *)view + delta, bytes};+ prefetch(GetCurrentProcess(), 1, &range, 0);+ UnmapViewOfFile(view);+ }+ CloseHandle(mapping);+ }+ }+ CloseHandle(job->file);+ free(job);+ InterlockedDecrement(&compat_prefetch_pending);+}++static inline int compat_prefetch_file(int fd, off_t off, off_t len) {+ if (fd < 0 || off < 0 || len <= 0) return 0;+ /* Drop excess hints instead of queueing stale experts or blocking inference.+ * The worker maps at most 64 MiB and allocates no read-sized scratch copy. */+ LONG n = InterlockedCompareExchange(&compat_prefetch_pending, 0, 0);+ for (;;) {+ if (n >= COMPAT_PREFETCH_JOBS) return 0;+ LONG was = InterlockedCompareExchange(&compat_prefetch_pending, n + 1, n);+ if (was == n) break;+ n = was;+ }+ intptr_t osfh = _get_osfhandle(fd);+ HANDLE file = NULL;+ HMODULE module = NULL;+ CompatPrefetchJob *job = NULL;+ if (osfh == -1 || osfh == -2 ||+ !DuplicateHandle(GetCurrentProcess(), (HANDLE)osfh, GetCurrentProcess(),+ &file, 0, FALSE, DUPLICATE_SAME_ACCESS)) goto fail;+ if (!GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS,+ (LPCSTR)(void *)compat_prefetch_run, &module)) goto fail;+ /* Executables cannot be unloaded; only a DLL needs deferred release. */+ if (module == GetModuleHandleA(NULL)) module = NULL;+ job = (CompatPrefetchJob *)malloc(sizeof(*job));+ if (!job) goto fail;+ *job = (CompatPrefetchJob){file, module, (unsigned long long)off,+ len > (off_t)COMPAT_PREFETCH_BYTES ? COMPAT_PREFETCH_BYTES : (SIZE_T)len};+ if (TrySubmitThreadpoolCallback(compat_prefetch_run, job, NULL)) return 0;+fail:+ if (file) CloseHandle(file);+ if (module) FreeLibrary(module);+ free(job);+ InterlockedDecrement(&compat_prefetch_pending);+ return 0; /* advice is never required for correctness */+}+#endifdiff --git a/c/tests/test_compat_direct.c b/c/tests/test_compat_direct.cindex ca9c7b45..897f186b 100644--- a/c/tests/test_compat_direct.c+++ b/c/tests/test_compat_direct.c@@ -51,8 +51,8 @@ int main(void){if(compat_open_direct("no_such_file.tmp")>=0) return fail("open missing file must fail");if(compat_fsize(-1)>=0) return fail("compat_fsize on bad fd must be negative");- /* compat_fadvise: WILLNEED warms the page cache (background read into throwaway- * buffer), DONTNEED is a documented no-op. After a WILLNEED the buffered fd's+ /* compat_fadvise: WILLNEED schedules bounded background readahead,+ * DONTNEED is a documented no-op. After a WILLNEED the buffered fd's* subsequent pread must still return the exact bytes β the cache-warmer must not* corrupt data. Bad fd / non-WILLNEED advice must be safe no-ops (return 0). */int wfd = open(TMPF, COMPAT_O_RDONLY);diff --git a/c/tests/test_compat_prefetch.c b/c/tests/test_compat_prefetch.cnew file mode 100644index 00000000..4c5eb8e2--- /dev/null+++ b/c/tests/test_compat_prefetch.c@@ -0,0 +1,144 @@+/* Windows readahead must never read weights on the caller, grow an unbounded+ * queue, or follow a recycled CRT descriptor. Real mappings and thread-pool+ * workers are used; a gate makes scheduling and resource failures deterministic. */+#include <stdio.h>+#ifndef _WIN32+int main(void) { puts("compat prefetch: skipped (native POSIX advice)"); return 0; }+#else+#include <stdlib.h>+#include <stdint.h>+#include <string.h>+#include <windows.h>+#include <io.h>+#include <fcntl.h>+#include <sys/types.h>+#include <sys/stat.h>++static HANDLE gate;+static volatile LONG submitted, completed, inspected, reads, bad;+static int reject_submit, reject_duplicate, missing_api;+static SIZE_T expected_bytes;+static unsigned char expected_value;+typedef struct { PVOID VirtualAddress; SIZE_T NumberOfBytes; } ProbeRange;+typedef BOOL (WINAPI *PrefetchFn)(HANDLE, ULONG_PTR, ProbeRange *, ULONG);++static BOOL WINAPI probe_prefetch(HANDLE process, ULONG_PTR count, ProbeRange *ranges, ULONG flags) {+ if (count != 1 || flags || ranges[0].NumberOfBytes != expected_bytes ||+ *(unsigned char *)ranges[0].VirtualAddress != expected_value)+ InterlockedIncrement(&bad);+ InterlockedIncrement(&inspected);+ PrefetchFn fn = (PrefetchFn)(void *)GetProcAddress(GetModuleHandleA("kernel32.dll"), "PrefetchVirtualMemory");+ return fn ? fn(process, count, ranges, flags) : TRUE;+}+static FARPROC WINAPI probe_get_proc(HMODULE module, LPCSTR name) {+ if (!strcmp(name, "PrefetchVirtualMemory"))+ return missing_api ? NULL : (FARPROC)(void *)probe_prefetch;+ return GetProcAddress(module, name);+}+typedef struct { PTP_SIMPLE_CALLBACK fn; PVOID arg; } ProbeWork;+static VOID CALLBACK probe_worker(PTP_CALLBACK_INSTANCE instance, PVOID arg) {+ ProbeWork work = *(ProbeWork *)arg; free(arg);+ if (WaitForSingleObject(gate, 10000) != WAIT_OBJECT_0) InterlockedIncrement(&bad);+ work.fn(instance, work.arg);+ InterlockedIncrement(&completed);+}+static BOOL WINAPI probe_submit(PTP_SIMPLE_CALLBACK fn, PVOID arg, PTP_CALLBACK_ENVIRON env) {+ if (reject_submit) return FALSE;+ ProbeWork *work = malloc(sizeof(*work));+ if (!work) return FALSE;+ *work = (ProbeWork){fn, arg};+ if (!TrySubmitThreadpoolCallback(probe_worker, work, env)) { free(work); return FALSE; }+ InterlockedIncrement(&submitted);+ return TRUE;+}+static BOOL WINAPI probe_duplicate(HANDLE a, HANDLE b, HANDLE c, LPHANDLE d, DWORD e, BOOL f, DWORD g) {+ return !reject_duplicate && DuplicateHandle(a, b, c, d, e, f, g);+}+static BOOL WINAPI probe_read(HANDLE a, LPVOID b, DWORD c, LPDWORD d, LPOVERLAPPED e) {+ InterlockedIncrement(&reads);+ return ReadFile(a, b, c, d, e);+}+#define GetProcAddress probe_get_proc+#define TrySubmitThreadpoolCallback probe_submit+#define DuplicateHandle probe_duplicate+#define ReadFile probe_read+#include "../compat.h"+#undef GetProcAddress+#undef TrySubmitThreadpoolCallback+#undef DuplicateHandle+#undef ReadFile++#define CHECK(x) do { if (!(x)) { fprintf(stderr, "FAIL line %d: %s\n", __LINE__, #x); return 1; } } while (0)+static int drain(void) {+ SetEvent(gate);+ ULONGLONG end = GetTickCount64() + 10000;+ while (InterlockedCompareExchange(&completed, 0, 0) != submitted && GetTickCount64() < end) Sleep(1);+ if (completed != submitted || compat_prefetch_pending || bad)+ fprintf(stderr, "drain: submitted=%ld completed=%ld pending=%ld inspected=%ld bad=%ld\n",+ submitted, completed, compat_prefetch_pending, inspected, bad);+ return completed == submitted && !compat_prefetch_pending && !bad;+}+int main(void) {+ gate = CreateEventW(NULL, TRUE, FALSE, NULL); CHECK(gate);+ const char *path = "test_prefetch.tmp";+ int fd = _open(path, _O_RDWR | _O_CREAT | _O_TRUNC | _O_BINARY, _S_IREAD | _S_IWRITE);+ CHECK(fd >= 0);+ unsigned char data[4096]; memset(data, 0x5a, sizeof data);+ CHECK(_write(fd, data, sizeof data) == sizeof data);+ expected_value = 0x5a; expected_bytes = 4001;+ for (int i = 0; i < 100; i++) CHECK(!posix_fadvise(fd, 95, 4001, POSIX_FADV_WILLNEED));+ CHECK(submitted == COMPAT_PREFETCH_JOBS);+ CHECK(compat_prefetch_pending == COMPAT_PREFETCH_JOBS);+ CHECK(reads == 0); /* no synchronous pre-read on the caller */+ CHECK(_lseeki64(fd, 0, SEEK_CUR) == sizeof data);+ int old = fd; _close(fd);+ fd = _open("test_prefetch_reused.tmp", _O_RDWR | _O_CREAT | _O_TRUNC | _O_BINARY, _S_IREAD | _S_IWRITE);+ CHECK(fd == old); /* queued work must retain the original file */+ memset(data, 0x33, sizeof data); CHECK(_write(fd, data, sizeof data) == sizeof data);+ CHECK(drain()); CHECK(inspected == COMPAT_PREFETCH_JOBS);+ CHECK(remove(path) == 0); /* no duplicate handle or mapping left open */++ DWORD handles_before, handles_after;+ CHECK(GetProcessHandleCount(GetCurrentProcess(), &handles_before));+ reject_submit = 1;+ for (int i = 0; i < 100; i++) CHECK(!posix_fadvise(fd, 0, 4096, POSIX_FADV_WILLNEED));+ reject_submit = 0; reject_duplicate = 1;+ for (int i = 0; i < 100; i++) CHECK(!posix_fadvise(fd, 0, 4096, POSIX_FADV_WILLNEED));+ reject_duplicate = 0;+ CHECK(!compat_prefetch_pending && submitted == COMPAT_PREFETCH_JOBS);+ CHECK(GetProcessHandleCount(GetCurrentProcess(), &handles_after));+ CHECK(handles_after <= handles_before);++ /* >4 GiB, unaligned offset and a hint crossing EOF. The sparse file avoids+ * allocating its hole on disk; only the final page is written. */+ HANDLE h = (HANDLE)_get_osfhandle(fd);+ DWORD got;+ CHECK(DeviceIoControl(h, FSCTL_SET_SPARSE, NULL, 0, NULL, 0, &got, NULL));+ off_t offset = ((off_t)1 << 32) + 8192 + 19;+ CHECK(_lseeki64(fd, offset, SEEK_SET) == offset);+ CHECK(_write(fd, data, sizeof data) == sizeof data);+ expected_value = 0x33; expected_bytes = sizeof data;+ CHECK(!posix_fadvise(fd, offset, 65536, POSIX_FADV_WILLNEED)); CHECK(drain());+ CHECK(inspected == COMPAT_PREFETCH_JOBS + 1);++ /* Capped window, missing API and empty/invalid ranges are only hints. */+ expected_bytes = COMPAT_PREFETCH_BYTES; expected_value = 0x33;+ CHECK(!posix_fadvise(fd, 0, offset, POSIX_FADV_WILLNEED)); CHECK(drain());+ LONG before = inspected;+ missing_api = 1;+ CHECK(!posix_fadvise(fd, 0, 4096, POSIX_FADV_WILLNEED)); CHECK(drain());+ missing_api = 0;+ CHECK(!posix_fadvise(fd, offset + 65536, 4096, POSIX_FADV_WILLNEED)); CHECK(drain());+ CHECK(inspected == before);+ LONG jobs = submitted;+ CHECK(!posix_fadvise(fd, 0, 4096, POSIX_FADV_DONTNEED));+ CHECK(!posix_fadvise(fd, -1, 4096, POSIX_FADV_WILLNEED));+ CHECK(!posix_fadvise(-1, 0, 4096, POSIX_FADV_WILLNEED));+ CHECK(!posix_fadvise(fd, 0, 0, POSIX_FADV_WILLNEED));+ CHECK(submitted == jobs && !reads);+ _close(fd); CHECK(!remove("test_prefetch_reused.tmp"));+ CloseHandle(gate);+ puts("compat prefetch: PASS (async, bounded, fd reuse, >4 GiB, EOF, failures)");+ return 0;+}+#endifdiff --git a/docs/windows.md b/docs/windows.mdindex ebf9e6c6..2308cd5d 100644--- a/docs/windows.md+++ b/docs/windows.md@@ -1,5 +1,14 @@# Windows 11 native install β a complete walkthrough (no WSL)+Expert readahead uses a bounded Windows thread-pool queue. A `WILLNEED` hint+returns without reading the expert on the inference thread, even when the file+was opened synchronously. The worker prefetches a read-only file mapping;+there is no second, temporary copy of the expert. At most four hints per engine+translation unit are pending, with each window capped at 64 MiB. Excess hints,+unavailable APIs and failed mappings are safely skipped; ordinary model reads+still supply the weights. This applies to the common file-advice path used by+the CPU and GPU engines and does not change their weight formats or arithmetic.+A start-to-finish, reproducible path from a fresh Windows 11 machine to GLM-5.2 generating tokens, with the GPU tier. Every step and every failure mode below was hit and verified on real hardware: Core Ultra 9 285K (AVX-VNNI) / RTX 5080 (sm_120) / 128 GB RAM / Windows 11 24H2 (issue #306). Steps are ordered so the long downloads run while you build.---