Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

bench(cuda): measure paired resident projection batching

ZacharyZcRaee54ce11273
diff --git a/c/Makefile b/c/Makefile
index 47eee120..7927e66e 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -956,6 +956,10 @@ rans: $(RANSLIB)
$(RANSLIB): tools/rans_ctypes.c rans.h
$(CC) $(CFLAGS) -fPIC -shared $< -o $@ $(LDFLAGS)
+# Matched resident int8 GPU projection timings; run explicitly, not in CI.
+tests/bench_cuda_resident_batch$(EXE): tests/bench_cuda_resident_batch.cu backend_cuda.cu backend_cuda.h backend_gpu_compat.h
+ "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/bench_cuda_resident_batch.cu -o $@ $(ANS_NVCC_LIBS) $(GPU_TEST_LIBS)
+
cuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backend_cuda.cu tests/test_ragged_attention.cu tests/test_absorb_determinism.cu tests/test_mxfp4_cuda.cu tests/mxfp4_ref.c tests/test_fp8_warp_cuda.cu tests/test_fp8_cuda.cu tests/test_weights_owned_cuda.cu tests/test_cuda_fmt_trap_cuda.cu tests/test_alloc_footprint_cuda.cu
@command -v "$(GPUCC)" >/dev/null 2>&1 || { echo "$(GPUCC_NAME) not found" >&2; exit 1; }
"$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) $(ANS_NVCC_LIBS) $(GPU_TEST_LIBS)
diff --git a/c/tests/bench_cuda_resident_batch.cu b/c/tests/bench_cuda_resident_batch.cu
new file mode 100644
index 00000000..2f911fbc
--- /dev/null
+++ b/c/tests/bench_cuda_resident_batch.cu
@@ -0,0 +1,80 @@
+/* Bounded resident-int8 projection microbenchmark; no model files required.
+ * Times synchronous host API calls (copies + compute), excluding upload.
+ * Compare S one-row calls with one S-row call on identical resident weights.
+ * This is not full-model prefill throughput or a CPU/GPU comparison. */
+#include "../backend_cuda.h"
+#include <algorithm>
+#include <chrono>
+#include <cmath>
+#include <cstdio>
+#include <vector>
+
+static double median(std::vector<double> values) {
+ std::sort(values.begin(),values.end());
+ return values[values.size()/2];
+}
+
+static bool measure(int I,int O,int S,int device) {
+ std::vector<int8_t> q((size_t)I*O);
+ std::vector<float> scale(O), x((size_t)S*I), serial((size_t)S*O), batch(serial.size());
+ for(size_t i=0;i<q.size();i++) q[i]=(int)((i*17+i/I*13)%31)-15;
+ for(int o=0;o<O;o++) scale[o]=(o%3+1)/128.f;
+ for(size_t i=0;i<x.size();i++) x[i]=(int(i%17)-8)/64.f;
+ ColiCudaTensor *tensor=nullptr;
+ if(!coli_cuda_tensor_upload(&tensor,q.data(),scale.data(),1,I,O,device)) return false;
+ auto run = [&](bool batched) {
+ float *out=batched?batch.data():serial.data();
+ if(batched) return coli_cuda_matmul(&tensor,out,x.data(),nullptr,nullptr,1,S,I,O,device,0)!=0;
+ for(int row=0;row<S;row++)
+ if(!coli_cuda_matmul(&tensor,out+(size_t)row*O,x.data()+(size_t)row*I,
+ nullptr,nullptr,1,1,I,O,device,0)) return false;
+ return true;
+ };
+ bool ok=true;
+ for(int i=0;i<2 && ok;i++) ok=run(false)&&run(true);
+ std::vector<double> times[2], ratios;
+ for(int rep=0;rep<9 && ok;rep++) {
+ double pair[2]={};
+ for(int arm=0;arm<2 && ok;arm++) {
+ int mode=(rep+arm)%2;
+ auto start=std::chrono::steady_clock::now();
+ ok=run(mode!=0);
+ pair[mode]=std::chrono::duration<double,std::milli>(std::chrono::steady_clock::now()-start).count();
+ times[mode].push_back(pair[mode]);
+ }
+ if(ok) ratios.push_back(pair[0]/pair[1]);
+ for(size_t i=0;i<serial.size() && ok;i++)
+ ok=std::isfinite(serial[i]) && std::isfinite(batch[i]) &&
+ std::fabs(serial[i]-batch[i])<=1e-5f*(1.f+std::fabs(serial[i]));
+ }
+ // Independent sampled CPU reference, outside timed regions. Full outputs
+ // are compared between arms after every pair, not only the final sample.
+ double max_error=0;
+ for(int row=0;row<S && ok;row+=std::max(1,S/7)) for(int o=0;o<O;o+=std::max(1,O/7)) {
+ double reference=0;
+ for(int i=0;i<I;i++) reference+=double(x[(size_t)row*I+i])*q[(size_t)o*I+i];
+ reference*=scale[o];
+ double error=std::fabs(batch[(size_t)row*O+o]-reference);
+ max_error=std::max(max_error,error);
+ if(error>1e-5*(1+std::fabs(reference))) ok=false;
+ }
+ coli_cuda_tensor_free(tensor);
+ if(!ok){std::fprintf(stderr,"FAIL: resident batch I=%d O=%d S=%d\n",I,O,S);return false;}
+ std::printf("{\"input\":%d,\"output\":%d,\"rows\":%d,\"pairs\":9,"
+ "\"serial_median_ms\":%.6f,\"batch_median_ms\":%.6f,"
+ "\"paired_speedup_median\":%.6f,\"cpu_sample_max_abs_error\":%.9g,"
+ "\"serial_ms\":[",I,O,S,median(times[0]),median(times[1]),median(ratios),max_error);
+ for(size_t i=0;i<times[0].size();i++) std::printf("%s%.6f",i?",":"",times[0][i]);
+ std::printf("],\"batch_ms\":[");
+ for(size_t i=0;i<times[1].size();i++) std::printf("%s%.6f",i?",":"",times[1][i]);
+ std::puts("]}");
+ return true;
+}
+
+int main() {
+ int device=0;
+ if(!coli_cuda_init(&device,1)) return 77;
+ bool ok=measure(512,1024,32,device) && measure(2048,2048,32,device) && measure(2048,2048,128,device);
+ coli_cuda_shutdown();
+ return ok?0:1;
+}
diff --git a/docs/experiments/qwen36-resident-batch-2026-09-22.jsonl b/docs/experiments/qwen36-resident-batch-2026-09-22.jsonl
new file mode 100644
index 00000000..1e93a53f
--- /dev/null
+++ b/docs/experiments/qwen36-resident-batch-2026-09-22.jsonl
@@ -0,0 +1,3 @@
+{"input":512,"output":1024,"rows":32,"pairs":9,"serial_median_ms":2.974592,"batch_median_ms":0.293992,"paired_speedup_median":10.018000,"cpu_sample_max_abs_error":0,"serial_ms":[2.879376,3.728909,2.975827,2.811392,2.950584,2.995621,2.974592,2.966765,3.317602],"batch_ms":[0.291110,0.293992,0.297048,0.313669,0.287397,0.288322,0.299809,0.318531,0.286065]}
+{"input":2048,"output":2048,"rows":32,"pairs":9,"serial_median_ms":3.588027,"batch_median_ms":0.812655,"paired_speedup_median":4.314296,"cpu_sample_max_abs_error":0,"serial_ms":[2.770069,4.095639,2.697552,3.588027,3.637298,4.933534,4.105374,2.849211,2.814900],"batch_ms":[0.793348,0.812655,0.815726,0.831660,0.812434,0.848347,0.814705,0.811320,0.785599]}
+{"input":2048,"output":2048,"rows":128,"pairs":9,"serial_median_ms":12.715961,"batch_median_ms":2.912792,"paired_speedup_median":4.365558,"cpu_sample_max_abs_error":0,"serial_ms":[12.715961,13.312398,13.366993,13.219180,15.129256,11.415760,11.160518,10.967418,11.883601],"batch_ms":[2.912792,2.887985,2.912167,2.882372,2.944258,2.923530,2.899905,3.928513,2.944615]}
diff --git a/docs/qwen36-cuda-tier.md b/docs/qwen36-cuda-tier.md
index 1b69011a..d45ab784 100644
--- a/docs/qwen36-cuda-tier.md
+++ b/docs/qwen36-cuda-tier.md
@@ -197,3 +197,46 @@ particular has no int8 analogue (see **Memory**).
CPU-only baseline of this engine before the tier: 0.35 tok/s.
Numerics: logits cosine vs the f32 CPU reference 0.9992 (dense int8 on),
bit-identical GPU-vs-CPU on the same container (cosine 1.0000001).
+
+## Resident projection batching microbenchmark
+
+Build and run a bounded, model-free comparison of `S` resident-int8 one-row GPU
+calls with one `S`-row GPU call:
+
+```sh
+make -C c tests/bench_cuda_resident_batch CUDA=1 CUDA_ARCH=sm_89
+c/tests/bench_cuda_resident_batch > resident-batch.jsonl
+```
+
+Set `CUDA_HOME` if the toolkit is not on PATH and select the architecture for
+your device. The harness uses CUDA device 0, uploads each matrix once, warms
+both paths twice, then alternates their order over nine paired measurements.
+The host wall time includes synchronous input/output copies and compute, but
+excludes upload and validation. All output elements are checked between arms
+after every pair; sampled elements also have an independent CPU reference.
+JSONL retains all nine timings per arm and the median of the paired ratios.
+An execution/validation failure exits nonzero; absent CUDA initialization exits
+77. The target is explicitly run and is not part of ordinary CI.
+
+A single run on 2026-09-22 (RTX 4070 12 GB, driver 591.86, CUDA 12.9, `sm_89`,
+Core Ultra 9 285K, WSL2 Linux 6.6.114.1, `-O3 -ftz=false`) produced:
+
+| Input × output | Rows | Serial median ms | Batch median ms | Median paired ratio |
+|---|---:|---:|---:|---:|
+| 512 × 1024 | 32 | 2.975 | 0.294 | 10.02 |
+| 2048 × 2048 | 32 | 3.588 | 0.813 | 4.31 |
+| 2048 × 2048 | 128 | 12.716 | 2.913 | 4.37 |
+
+[Raw paired timings](experiments/qwen36-resident-batch-2026-09-22.jsonl)
+include zero sampled CPU-reference error for these deterministic synthetic
+inputs. The device was not isolated: telemetry before/after showed 4% GPU
+utilization, approximately 3 GB occupied VRAM and 2550 MHz SM clock. Timings show
+visible variation; one nine-pair run does not establish reproducibility across
+sessions, devices or real activation distributions.
+
+The serial GPU baseline reflects the projection-call pattern replaced by the
+DeltaNet input batching change. These are synthetic shapes, not complete
+DeltaNet layers: convolution, recurrent updates, normalization, other projections,
+model loading and HTTP queueing are excluded. This is **not** an end-to-end model
+speedup, nor an attention-prefill speedup: the old attention prefill path used
+CPU batch matmul, which is not an arm in this benchmark.