Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
test(windows): verify queued prefetch survives DLL release
JustVuggad157c8f22f8
diff --git a/c/Makefile b/c/Makefileindex 2201d591..5960a125 100644--- a/c/Makefile+++ b/c/Makefile@@ -2115,9 +2115,12 @@ tests/test_i4_acc512$(EXE): tests/test_i4_acc512.ctests/test_compat_direct$(EXE): tests/test_compat_direct.c$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)-tests/test_compat_prefetch$(EXE): tests/test_compat_prefetch.c+tests/test_compat_prefetch$(EXE): tests/test_compat_prefetch.c $(if $(IS_WIN),tests/compat_prefetch_module.dll)$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+tests/compat_prefetch_module.dll: tests/compat_prefetch_module.c+ $(CC) $(CFLAGS) -shared $< -o $@ $(LDFLAGS)+tests/test_expert_store_ops$(EXE): tests/test_expert_store_ops.c$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)@@ -2873,7 +2876,7 @@ AUTODEP_OBJS = backend_vulkan.o vk_tier.o backend_loader.o backend_xdna.o qAUTODEP_BINS = $(addsuffix $(EXE),$(ENGINE_RULES)) \$(addprefix tests/,$(addsuffix $(EXE),$(filter-out $(AUTODEP_ELSEWHERE) \$(AUTODEP_OWN_FLAGS),$(AUTODEP_TESTS)))) \- $(AUTODEP_OBJS)+ $(AUTODEP_OBJS) $(if $(IS_WIN),tests/compat_prefetch_module.dll)AUTODEP_DEPS = $(addsuffix .d,$(basename $(AUTODEP_BINS)))$(foreach b,$(AUTODEP_BINS),$(eval $(b): $(basename $(b)).d))$(AUTODEP_DEPS): ;diff --git a/c/tests/compat_prefetch_module.c b/c/tests/compat_prefetch_module.cnew file mode 100644index 00000000..2889772b--- /dev/null+++ b/c/tests/compat_prefetch_module.c@@ -0,0 +1,29 @@+/* Loaded and released while an advisory worker is running: an SDK caller+ * must not be able to unload the callback's code before it returns. */+#ifdef _WIN32+#include <windows.h>+#include <stdio.h>+#include <string.h>+#include <io.h>+#include <fcntl.h>+#include <sys/types.h>+static HANDLE release_gate, entered_gate;+static FARPROC WINAPI delayed_get_proc(HMODULE module, LPCSTR name) {+ if (!strcmp(name, "PrefetchVirtualMemory")) {+ SetEvent(entered_gate);+ if (WaitForSingleObject(release_gate, 10000) != WAIT_OBJECT_0) return NULL;+ }+ return GetProcAddress(module, name);+}+#define GetProcAddress delayed_get_proc+#include "../compat.h"+#undef GetProcAddress+__declspec(dllexport) int start_prefetch(const char *path, HANDLE release, HANDLE entered) {+ release_gate = release; entered_gate = entered;+ int fd = _open(path, _O_RDONLY | _O_BINARY);+ if (fd < 0) return 0;+ compat_fadvise(fd, 0, 4096, POSIX_FADV_WILLNEED);+ _close(fd);+ return compat_prefetch_pending > 0;+}+#endifdiff --git a/c/tests/test_compat_prefetch.c b/c/tests/test_compat_prefetch.cindex 4c5eb8e2..9a360f4b 100644--- a/c/tests/test_compat_prefetch.c+++ b/c/tests/test_compat_prefetch.c@@ -136,9 +136,31 @@ int main(void) {CHECK(!posix_fadvise(-1, 0, 4096, POSIX_FADV_WILLNEED));CHECK(!posix_fadvise(fd, 0, 0, POSIX_FADV_WILLNEED));CHECK(submitted == jobs && !reads);- _close(fd); CHECK(!remove("test_prefetch_reused.tmp"));+ _close(fd);+ /* The helper lives beside this executable, independent of the test CWD. */+ char library[MAX_PATH];+ DWORD length = GetModuleFileNameA(NULL, library, sizeof library);+ CHECK(length && length < sizeof library);+ char *slash = strrchr(library, '\\'); CHECK(slash);+ CHECK((size_t)(slash - library) + sizeof("\\compat_prefetch_module.dll") < sizeof library);+ strcpy(slash + 1, "compat_prefetch_module.dll");+ HANDLE entered = CreateEventW(NULL, TRUE, FALSE, NULL); CHECK(entered);+ ResetEvent(gate);+ HMODULE module = LoadLibraryA(library); CHECK(module);+ typedef int (*StartPrefetch)(const char *, HANDLE, HANDLE);+ StartPrefetch start = (StartPrefetch)(void *)GetProcAddress(module, "start_prefetch");+ CHECK(start && start("test_prefetch_reused.tmp", gate, entered));+ CHECK(WaitForSingleObject(entered, 10000) == WAIT_OBJECT_0);+ CHECK(FreeLibrary(module));+ CHECK(GetModuleHandleA("compat_prefetch_module.dll"));+ SetEvent(gate);+ ULONGLONG end = GetTickCount64() + 10000;+ while (GetModuleHandleA("compat_prefetch_module.dll") && GetTickCount64() < end) Sleep(1);+ CHECK(!GetModuleHandleA("compat_prefetch_module.dll"));+ CloseHandle(entered);+ CHECK(!remove("test_prefetch_reused.tmp"));CloseHandle(gate);- puts("compat prefetch: PASS (async, bounded, fd reuse, >4 GiB, EOF, failures)");+ puts("compat prefetch: PASS (async, bounded, fd reuse, >4 GiB, EOF, failures, DLL lifetime)");return 0;}#endifdiff --git a/c/tools/clean.py b/c/tools/clean.pyindex 2be87210..578d4864 100644--- a/c/tools/clean.py+++ b/c/tools/clean.py@@ -37,6 +37,7 @@ FILES = ["coli_cuda.dll", "coli_cuda.lib", "coli_cuda.exp",# hipcc emits an import library, export file and PDB alongside the DLL."coli_hip.dll", "coli_hip.lib", "coli_hip.exp", "coli_hip.pdb",+ "tests/compat_prefetch_module.dll","deepseek_v4", "deepseek_v4.exe", "deepseek_v4.cflags", "deepseek_v4.cudaflags","deepseek_v41", "deepseek_v41.exe","mimo", "mimo.exe",diff --git a/docs/windows.md b/docs/windows.mdindex 2308cd5d..faaa2bdd 100644--- a/docs/windows.md+++ b/docs/windows.md@@ -1,14 +1,5 @@# Windows 11 native install — a complete walkthrough (no WSL)-Expert readahead uses a bounded Windows thread-pool queue. A `WILLNEED` hint-returns without reading the expert on the inference thread, even when the file-was opened synchronously. The worker prefetches a read-only file mapping;-there is no second, temporary copy of the expert. At most four hints per engine-translation unit are pending, with each window capped at 64 MiB. Excess hints,-unavailable APIs and failed mappings are safely skipped; ordinary model reads-still supply the weights. This applies to the common file-advice path used by-the CPU and GPU engines and does not change their weight formats or arithmetic.-A start-to-finish, reproducible path from a fresh Windows 11 machine to GLM-5.2 generating tokens, with the GPU tier. Every step and every failure mode below was hit and verified on real hardware: Core Ultra 9 285K (AVX-VNNI) / RTX 5080 (sm_120) / 128 GB RAM / Windows 11 24H2 (issue #306). Steps are ordered so the long downloads run while you build.---@@ -696,3 +687,14 @@ Remove-Item -Recurse -Force $toolingSDK paths containing spaces are supported — every SDK-derived path the recipepasses to the compiler is quoted. If you supply one yourself, quote it:`HIP_SDK_ROOT="<path with spaces>"`.++## Expert readahead++Expert readahead uses a bounded Windows thread-pool queue. A `WILLNEED` hint+returns without reading the expert on the inference thread, even when the file+was opened synchronously. The worker prefetches a read-only file mapping;+there is no second, temporary copy of the expert. At most four hints per engine+translation unit are pending, with each window capped at 64 MiB. Excess hints,+unavailable APIs and failed mappings are safely skipped; ordinary model reads+still supply the weights. This applies to the common file-advice path used by+the CPU and GPU engines and does not change their weight formats or arithmetic.