Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

cuda: forward device telemetry through Windows loader

ZacharyZcRde4aa66bae5e
diff --git a/c/backend_loader.c b/c/backend_loader.c
index 434a4a8c..e2192039 100644
--- a/c/backend_loader.c
+++ b/c/backend_loader.c
@@ -37,6 +37,10 @@ typedef int (*fn_device_integrated)(int device);
typedef void (*fn_stats)(int device, size_t *tensor_count, size_t *tensor_bytes);
typedef void (*fn_group_stats)(uint64_t *calls, uint64_t *experts, uint64_t *rows,
double *h2d_ms, double *kernel_ms, double *d2h_ms);
+typedef void (*fn_group_stats_device)(int device, uint64_t *calls,
+ uint64_t *experts, uint64_t *rows,
+ double *h2d_ms, double *kernel_ms,
+ double *d2h_ms);
typedef int (*fn_expert_mlp)(ColiCudaTensor *gate, ColiCudaTensor *up,
ColiCudaTensor *down, float *y, const float *x, int S);
typedef int (*fn_expert_group)(ColiCudaTensor *const *gates, ColiCudaTensor *const *ups,
@@ -110,6 +114,7 @@ static struct {
fn_device_integrated device_integrated;
fn_stats stats;
fn_group_stats group_stats;
+ fn_group_stats_device group_stats_device;
fn_expert_mlp expert_mlp;
fn_expert_group expert_group;
fn_expert_group_issue expert_group_issue;
@@ -224,6 +229,7 @@ static int coli_cuda_load(void){
RESOLVE_OPT(device_integrated, fn_device_integrated)
RESOLVE(stats, fn_stats)
RESOLVE(group_stats, fn_group_stats)
+ RESOLVE(group_stats_device, fn_group_stats_device)
RESOLVE(expert_mlp, fn_expert_mlp)
RESOLVE(expert_group, fn_expert_group)
RESOLVE(expert_group_issue, fn_expert_group_issue)
@@ -320,6 +326,17 @@ void coli_cuda_group_stats(uint64_t *calls, uint64_t *experts, uint64_t *rows,
g_cuda.group_stats(calls, experts, rows, h2d_ms, kernel_ms, d2h_ms);
}
+void coli_cuda_group_stats_device(int device, uint64_t *calls, uint64_t *experts,
+ uint64_t *rows, double *h2d_ms,
+ double *kernel_ms, double *d2h_ms){
+ if(!g_cuda.available){
+ if(calls)*calls=0; if(experts)*experts=0; if(rows)*rows=0;
+ if(h2d_ms)*h2d_ms=0; if(kernel_ms)*kernel_ms=0; if(d2h_ms)*d2h_ms=0;
+ return;
+ }
+ g_cuda.group_stats_device(device, calls, experts, rows, h2d_ms, kernel_ms, d2h_ms);
+}
+
int coli_cuda_expert_mlp(ColiCudaTensor *gate, ColiCudaTensor *up,
ColiCudaTensor *down, float *y, const float *x, int S){
if(!g_cuda.available) return 0;