Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
4.1 KB
Raw Blame
/* Bounded, advisory file readahead for Windows. Included by compat.h after
 * windows.h. No model pointers or CRT descriptors escape into a worker. */
#ifndef COLI_COMPAT_PREFETCH_WIN_H
#define COLI_COMPAT_PREFETCH_WIN_H

#define COMPAT_PREFETCH_JOBS 4
#define COMPAT_PREFETCH_BYTES (64u * 1024u * 1024u)

/* Keep the Windows 7 build target usable: the hint is optional on systems
 * without PrefetchVirtualMemory. This is its Windows 8 range ABI. */
typedef struct { PVOID VirtualAddress; SIZE_T NumberOfBytes; } CompatPrefetchRange;
typedef BOOL (WINAPI *CompatPrefetchFn)(HANDLE, ULONG_PTR, CompatPrefetchRange *, ULONG);
typedef struct {
    HANDLE file;
    HMODULE module;
    unsigned long long offset;
    SIZE_T bytes;
} CompatPrefetchJob;
static volatile LONG compat_prefetch_pending;

static VOID CALLBACK compat_prefetch_run(PTP_CALLBACK_INSTANCE instance, PVOID context) {
    CompatPrefetchJob *job = (CompatPrefetchJob *)context;
    /* An SDK DLL may be released as soon as its caller returns. The worker
     * owns a module reference until the thread pool has left this callback. */
    if (job->module) FreeLibraryWhenCallbackReturns(instance, job->module);
    CallbackMayRunLong(instance);
    CompatPrefetchFn prefetch = (CompatPrefetchFn)(void *)GetProcAddress(
        GetModuleHandleA("kernel32.dll"), "PrefetchVirtualMemory");
    LARGE_INTEGER size;
    if (prefetch && GetFileSizeEx(job->file, &size) && size.QuadPart > 0 &&
        job->offset < (unsigned long long)size.QuadPart) {
        unsigned long long left = (unsigned long long)size.QuadPart - job->offset;
        SIZE_T bytes = left < job->bytes ? (SIZE_T)left : job->bytes;
        SYSTEM_INFO info; GetSystemInfo(&info);
        unsigned long long base = job->offset - job->offset % info.dwAllocationGranularity;
        SIZE_T delta = (SIZE_T)(job->offset - base);
        HANDLE mapping = CreateFileMappingW(job->file, NULL, PAGE_READONLY, 0, 0, NULL);
        if (mapping) {
            void *view = MapViewOfFile(mapping, FILE_MAP_READ, (DWORD)(base >> 32),
                                      (DWORD)base, delta + bytes);
            if (view) {
                CompatPrefetchRange range = {(char *)view + delta, bytes};
                prefetch(GetCurrentProcess(), 1, &range, 0);
                UnmapViewOfFile(view);
            }
            CloseHandle(mapping);
        }
    }
    CloseHandle(job->file);
    free(job);
    InterlockedDecrement(&compat_prefetch_pending);
}

static inline int compat_prefetch_file(int fd, off_t off, off_t len) {
    if (fd < 0 || off < 0 || len <= 0) return 0;
    /* Drop excess hints instead of queueing stale experts or blocking inference.
     * The worker maps at most 64 MiB and allocates no read-sized scratch copy. */
    LONG n = InterlockedCompareExchange(&compat_prefetch_pending, 0, 0);
    for (;;) {
        if (n >= COMPAT_PREFETCH_JOBS) return 0;
        LONG was = InterlockedCompareExchange(&compat_prefetch_pending, n + 1, n);
        if (was == n) break;
        n = was;
    }
    intptr_t osfh = _get_osfhandle(fd);
    HANDLE file = NULL;
    HMODULE module = NULL;
    CompatPrefetchJob *job = NULL;
    if (osfh == -1 || osfh == -2 ||
        !DuplicateHandle(GetCurrentProcess(), (HANDLE)osfh, GetCurrentProcess(),
                         &file, 0, FALSE, DUPLICATE_SAME_ACCESS)) goto fail;
    if (!GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS,
                           (LPCSTR)(void *)compat_prefetch_run, &module)) goto fail;
    /* Executables cannot be unloaded; only a DLL needs deferred release. */
    if (module == GetModuleHandleA(NULL)) module = NULL;
    job = (CompatPrefetchJob *)malloc(sizeof(*job));
    if (!job) goto fail;
    *job = (CompatPrefetchJob){file, module, (unsigned long long)off,
        len > (off_t)COMPAT_PREFETCH_BYTES ? COMPAT_PREFETCH_BYTES : (SIZE_T)len};
    if (TrySubmitThreadpoolCallback(compat_prefetch_run, job, NULL)) return 0;
fail:
    if (file) CloseHandle(file);
    if (module) FreeLibrary(module);
    free(job);
    InterlockedDecrement(&compat_prefetch_pending);
    return 0;                       /* advice is never required for correctness */
}
#endif