Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

security: harden model-file parsing, server, and build per audit

KingIcyCreamProjects72dd9ac35db4
diff --git a/c/Makefile b/c/Makefile
index 7210cbbb..a9315012 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -143,16 +143,32 @@ else
CUDA_HOME ?= /usr/local/cuda
NVCC ?= $(CUDA_HOME)/bin/nvcc
endif
+# CUDA target selection. Default `native` builds SASS for the build machine's GPU
+# only (fast local dev). A headless CI/Docker build has no GPU, so `-arch=native`
+# there detects nothing and the shipped coli_cuda.dll fails at first kernel dispatch.
+# For a portable/release DLL that runs on any Ampere..Blackwell card AND JITs on
+# newer archs, build with CUDA_ARCH=portable: it emits SASS for sm_80/86/89/90/120
+# plus a compute_120 PTX fallback (sm_120 / RTX 50-series needs CUDA >= 12.8).
CUDA_ARCH ?= native
+ifeq ($(CUDA_ARCH),portable)
+CUDA_GENCODE = -gencode arch=compute_80,code=sm_80 \
+ -gencode arch=compute_86,code=sm_86 \
+ -gencode arch=compute_89,code=sm_89 \
+ -gencode arch=compute_90,code=sm_90 \
+ -gencode arch=compute_120,code=sm_120 \
+ -gencode arch=compute_120,code=compute_120
+else
+CUDA_GENCODE = -arch=$(CUDA_ARCH)
+endif
ifneq ($(IS_WIN),)
# nvcc's host compiler on Windows is MSVC cl.exe: the GCC-style
# -Xcompiler=-Wall,-Wextra is rejected ("D8021 invalid numeric argument
# '/Wextra'"), which made `make cuda-dll` unbuildable as shipped. -W3 is
# the MSVC warning level (dash form, NOT /W3: MSYS make would mangle the
# slash-form into a filesystem path).
-NVCCFLAGS ?= -O3 -std=c++17 -arch=$(CUDA_ARCH) -Xcompiler=-W3
+NVCCFLAGS ?= -O3 -std=c++17 $(CUDA_GENCODE) -Xcompiler=-W3
else
-NVCCFLAGS ?= -O3 -std=c++17 -arch=$(CUDA_ARCH) -Xcompiler=-Wall,-Wextra
+NVCCFLAGS ?= -O3 -std=c++17 $(CUDA_GENCODE) -Xcompiler=-Wall,-Wextra
endif
# PYTHON is a HOST tool (clean/test-c/test-python run it on the build machine),
# so key it off the host, NOT $(IS_WIN) — which is derived from the *target*
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cu
index 188330d9..ec1b3214 100644
--- a/c/backend_cuda.cu
+++ b/c/backend_cuda.cu
@@ -646,7 +646,8 @@ extern "C" int coli_cuda_expert_group(ColiCudaTensor *const *gates,
if(!cuda_ok(copy_desc,"expert group descriptors"))return 0;
int profile=getenv("COLI_CUDA_PROFILE")&&atoi(getenv("COLI_CUDA_PROFILE"));
cudaEvent_t ev[4]={};
- if(profile) for(int i=0;i<4;i++) if(!cuda_ok(cudaEventCreate(&ev[i]),"profile event")) profile=0;
+ if(profile) for(int i=0;i<4;i++) if(!cuda_ok(cudaEventCreate(&ev[i]),"profile event")){
+ for(int j=0;j<i;j++) cudaEventDestroy(ev[j]); profile=0; break; } /* (#B8) don't leak the events already created */
if(profile) cudaEventRecord(ev[0],ctx->stream);
if(async)std::memcpy(ctx->host_x,x,xb);
cudaError_t copy_x=async?cudaMemcpyAsync(ctx->x,ctx->host_x,xb,cudaMemcpyHostToDevice,ctx->stream)
diff --git a/c/download_fp8.py b/c/download_fp8.py
index fc8e2b23..dd4e96db 100644
--- a/c/download_fp8.py
+++ b/c/download_fp8.py
@@ -53,14 +53,14 @@ def download_file_ms(fn):
model_id=REPO_MS,
file_path=fn,
local_dir=DEST,
- revision="master",
+ revision=os.environ.get("GLM_MS_REVISION", "master"), # pin to a commit for supply-chain integrity
)
def download_file_hf(fn):
"""Download a single file from HuggingFace with hf_transfer."""
os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
from huggingface_hub import hf_hub_download
- hf_hub_download(REPO_HF, fn, local_dir=DEST)
+ hf_hub_download(REPO_HF, fn, local_dir=DEST, revision=os.environ.get("GLM_HF_REVISION", "main"))
def download_file_curl(fn, base_url, expected_size):
"""Fallback: download with curl to a .part file with resume."""
diff --git a/c/glm.c b/c/glm.c
index 70da3201..4043d8f1 100644
--- a/c/glm.c
+++ b/c/glm.c
@@ -1445,6 +1445,25 @@ static int detect_group_size(int O, int I, int64_t ns){
return 0;
}
+/* Reject a quantized tensor whose on-disk weight/scale byte counts don't EXACTLY
+ * match the config-derived [O,I] shape for the detected fmt. The fmt is inferred
+ * from the weight byte count and falls to int2 (fmt=3) on any mismatch; without
+ * this guard a crafted `.qs` (untrusted mirror, the documented threat model) picks
+ * a wrong fmt and the SIMD kernels index the weight slab / per-row scale buffer
+ * past its real length (OOB read), and qt_from_disk's O-sized scale buffer is
+ * overrun on read (OOB write). ns is the scale-tensor byte count (F32 scales). */
+static void qt_check_fmt(const char *name, int fmt, int gs, int O, int I, int64_t nb, int64_t ns){
+ int64_t ew, es; /* expected weight bytes, expected scale float count */
+ if(fmt==1){ ew=(int64_t)O*I; es=(int64_t)O; }
+ else if(fmt==2){ ew=(int64_t)O*((I+1)/2); es=(int64_t)O; }
+ else if(fmt==3){ ew=(int64_t)O*((I+3)/4); es=(int64_t)O; }
+ else if(fmt==4){ int ng=(I+gs-1)/gs; ew=(int64_t)O*((I+1)/2); es=(int64_t)O*ng; }
+ else { fprintf(stderr,"%s: unknown quant fmt %d\n",name,fmt); exit(1); }
+ if(nb!=ew || ns!=es*4){
+ fprintf(stderr,"%s: quantized size mismatch fmt=%d [O=%d,I=%d,gs=%d]: weight %lld (want %lld), scale %lld (want %lld)\n",
+ name,fmt,O,I,gs,(long long)nb,(long long)ew,(long long)ns,(long long)(es*4)); exit(1); }
+}
+
/* costruisce un QT [O,I] dal disco in `t` (buffer riusabili tra chiamate).
* - se esiste `name.qs`: pesi GIA' quantizzati nel container (U8 qdata + F32 scala) -> letti diretti
* - altrimenti: tensore pieno (f32/bf16) -> quantizzato a runtime a `bits` (oracolo tiny / pesi pieni)
@@ -1460,16 +1479,17 @@ static void qt_from_disk(Model *m, const char *name, int O, int I, int bits, int
int gs=0;
if(fmt==2) gs=detect_group_size(O,I,ns);
if(gs>0) fmt=4;
+ qt_check_fmt(name,fmt,gs,O,I,nb,ns); /* reject size/format mismatch before alloc+read */
if(fmt==1){ if(t->fmt!=1||!t->q8){ t->fmt=1; t->O=O; t->I=I; t->gs=0; t->q8=qalloc(nb); t->s=qsalloc(O); } st_read_raw(&m->S,name,t->q8,drop); }
else if(fmt==4){ int ng=(I+gs-1)/gs;
if(t->fmt!=4||!t->q4){ t->fmt=4; t->O=O; t->I=I; t->gs=gs; t->q4=qalloc(nb); t->s=falloc((int64_t)O*ng); }
st_read_raw(&m->S,name,t->q4,drop); }
else { if(t->fmt!=fmt||!t->q4){ t->fmt=fmt; t->O=O; t->I=I; t->gs=0; t->q4=qalloc(nb); t->s=qsalloc(O); } st_read_raw(&m->S,name,t->q4,drop); }
- st_read_f32(&m->S,sn,t->s,drop);
+ st_read_f32_cap(&m->S,sn,t->s, fmt==4?(int64_t)O*((I+gs-1)/gs):(int64_t)O, drop);
} else {
if(!t->qf && !t->q8 && !t->q4) qt_alloc(t,O,I,bits);
- if(t->fmt==0) st_read_f32(&m->S,name,t->qf,drop);
- else { float *tmp=falloc((int64_t)O*I); st_read_f32(&m->S,name,tmp,drop); qt_fill(t,tmp,bits); free(tmp); }
+ if(t->fmt==0) st_read_f32_cap(&m->S,name,t->qf,(int64_t)O*I,drop);
+ else { float *tmp=falloc((int64_t)O*I); st_read_f32_cap(&m->S,name,tmp,(int64_t)O*I,drop); qt_fill(t,tmp,bits); free(tmp); }
}
}
static QT qt_load(Model *m, const char *name, int O, int I, int bits){
@@ -1678,6 +1698,7 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits
/* embed: dequantizza la riga del token (scala per-riga) in x[hidden] */
static void embed_row(Model *m, int tok, float *x){
int D=m->c.hidden; QT *e=&m->embed;
+ if(tok<0 || tok>=e->O){ memset(x,0,(size_t)D*sizeof(float)); return; } /* #SEC-5: out-of-range token id -> zero row, never OOB */
if(e->fmt==0){ memcpy(x, e->qf+(int64_t)tok*D, D*sizeof(float)); return; }
if(e->fmt==1){ const int8_t *q=e->q8+(int64_t)tok*D; float s=e->s[tok];
for(int i=0;i<D;i++) x[i]=(float)q[i]*s; return; }
@@ -1807,6 +1828,7 @@ static int expert_load_impl(Model *m, int layer, int eid, ESlot *s, int fatal){
int gs=0;
if(fmt==2) gs=detect_group_size(OO[k],II[k],tq[k]->nbytes);
if(gs>0) fmt=4;
+ qt_check_fmt(nm[k],fmt,gs,OO[k],II[k],nb,tq[k]->nbytes); /* reject crafted size/format mismatch */
qt[k]->fmt=fmt; qt[k]->O=OO[k]; qt[k]->I=II[k]; qt[k]->gs=gs; qt[k]->qf=NULL;
qt[k]->q8=(int8_t*)((char*)bw[k]+tw[k]->off); qt[k]->q4=(uint8_t*)((char*)bw[k]+tw[k]->off);
qt[k]->s=(float*)((char*)bq[k]+tq[k]->off);
@@ -1935,6 +1957,7 @@ static int expert_load_impl(Model *m, int layer, int eid, ESlot *s, int fatal){
int gs=0;
if(fmt==2) gs=detect_group_size(OO[k],II[k],tq[k]->nbytes);
if(gs>0) fmt=4;
+ qt_check_fmt(nm[k],fmt,gs,OO[k],II[k],nb,tq[k]->nbytes); /* reject crafted size/format mismatch */
qt[k]->fmt=fmt; qt[k]->O=OO[k]; qt[k]->I=II[k]; qt[k]->gs=gs; qt[k]->qf=NULL;
qt[k]->q8=(int8_t*)(s->slab+pos[k]); qt[k]->q4=s->slab+pos[k]; qt[k]->s=fp[k];
}
diff --git a/c/json.h b/c/json.h
index 8a35e244..ef590536 100644
--- a/c/json.h
+++ b/c/json.h
@@ -55,10 +55,12 @@ static jval *j_parse_val(jparser *p);
static char *j_parse_str_raw(jparser *p) {
/* assume *p->s == '"' */
p->s++;
- const char *start = p->s;
- /* trova la fine gestendo gli escape, poi copia decodificando i casi base */
- char tmp[1 << 16]; int n = 0;
- #define J_PUT(ch) do{ if (n < (int)sizeof(tmp)-1) tmp[n++] = (char)(ch); }while(0)
+ /* buffer su heap che CRESCE: niente troncamento silenzioso a 64KB (le stringhe
+ * lunghe di tokenizer.json/config venivano tagliate) e niente 64KB di stack. */
+ size_t cap = 64, n = 0; char *tmp = (char *)malloc(cap);
+ if (!tmp) { fprintf(stderr, "OOM parsing JSON string\n"); exit(1); }
+ #define J_PUT(ch) do{ if (n + 1 >= cap) { cap *= 2; tmp = (char *)realloc(tmp, cap); \
+ if (!tmp) { fprintf(stderr, "OOM parsing JSON string\n"); exit(1); } } tmp[n++] = (char)(ch); }while(0)
while (*p->s && *p->s != '"') {
char c = *p->s++;
if (c == '\\' && *p->s) {
@@ -69,9 +71,11 @@ static char *j_parse_str_raw(jparser *p) {
case 'f': c = '\f'; break; case '/': c = '/'; break;
case '\\': c = '\\'; break; case '"': c = '"'; break;
case 'u': { /* \uXXXX -> codepoint UTF-8 (con coppie surrogate) */
+ if (!p->s[0]||!p->s[1]||!p->s[2]||!p->s[3]) { c='?'; break; } /* \u troncato: non leggere oltre il NUL */
unsigned cp = (unsigned)strtoul((char[]){p->s[0],p->s[1],p->s[2],p->s[3],0}, NULL, 16);
p->s += 4;
- if (cp >= 0xD800 && cp <= 0xDBFF && p->s[0]=='\\' && p->s[1]=='u') {
+ if (cp >= 0xD800 && cp <= 0xDBFF && p->s[0]=='\\' && p->s[1]=='u'
+ && p->s[2] && p->s[3] && p->s[4] && p->s[5]) {
unsigned lo = (unsigned)strtoul((char[]){p->s[2],p->s[3],p->s[4],p->s[5],0}, NULL, 16);
if (lo >= 0xDC00 && lo <= 0xDFFF) { cp = 0x10000 + ((cp-0xD800)<<10) + (lo-0xDC00); p->s += 6; }
}
@@ -88,8 +92,8 @@ static char *j_parse_str_raw(jparser *p) {
}
#undef J_PUT
if (*p->s == '"') p->s++;
- (void)start;
- return j_dup(p, tmp, n);
+ char *out = j_dup(p, tmp, (int)n); free(tmp);
+ return out;
}
static jval *j_parse_val(jparser *p) {
@@ -135,9 +139,9 @@ static jval *j_parse_val(jparser *p) {
p->depth--;
return v;
}
- if (c == 't') { p->s += 4; jval *v = j_new(J_BOOL); v->boolean = 1; return v; }
- if (c == 'f') { p->s += 5; jval *v = j_new(J_BOOL); v->boolean = 0; return v; }
- if (c == 'n') { p->s += 4; return j_new(J_NULL); }
+ if (c == 't' && !strncmp(p->s, "true", 4)) { p->s += 4; jval *v = j_new(J_BOOL); v->boolean = 1; return v; }
+ if (c == 'f' && !strncmp(p->s, "false", 5)) { p->s += 5; jval *v = j_new(J_BOOL); v->boolean = 0; return v; }
+ if (c == 'n' && !strncmp(p->s, "null", 4)) { p->s += 4; return j_new(J_NULL); }
/* numero */
{ char *end; double d = strtod(p->s, &end); p->s = end; jval *v = j_new(J_NUM); v->num = d; return v; }
}
diff --git a/c/openai_server.py b/c/openai_server.py
index d55fbc84..94095d63 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -86,6 +86,7 @@ class GenerationScheduler:
@contextlib.contextmanager
def admit(self, cancelled=None, slot=None):
ticket = object()
+ entry = (ticket, slot) # (#B2) remember each waiter's target slot for fair, per-slot admission
queued_at = time.monotonic()
with self.condition:
if self.closed:
@@ -95,31 +96,46 @@ class GenerationScheduler:
self.rejected += 1
raise APIError(429, "The inference queue is full.", None, "queue_full",
"rate_limit_error", {"Retry-After": "1"})
- self.queue.append(ticket)
+ self.queue.append(entry)
deadline = queued_at + self.queue_timeout
while True:
if self.closed:
- self.queue.remove(ticket)
+ self.queue.remove(entry)
self.condition.notify_all()
raise APIError(503, "The inference scheduler is shutting down.", None,
"scheduler_closed", "server_error")
available = min(self.free_slots) if slot is None and self.free_slots else slot
- if self.queue[0] is ticket and available in self.free_slots:
+ # (#B2) Admit as soon as our target slot is free AND no strictly-earlier
+ # waiter also wants it (an earlier waiter "wants" it if it is any-slot or
+ # pinned to the same slot). This replaces the old strict FIFO-head rule,
+ # which let a head pinned to a busy slot block every request behind it —
+ # even ones targeting a currently-free slot (head-of-line blocking).
+ # ponytail: O(queue) scan per wakeup — negligible at the default max_queue;
+ # switch to per-slot wait sets if max_queue is ever raised to thousands.
+ can_admit = available in self.free_slots
+ if can_admit:
+ for t2, s2 in self.queue:
+ if t2 is ticket:
+ break
+ if s2 is None or s2 == available:
+ can_admit = False
+ break
+ if can_admit:
break
if cancelled and cancelled():
- self.queue.remove(ticket)
+ self.queue.remove(entry)
self.cancelled += 1
self.condition.notify_all()
raise ClientCancelled()
remaining = deadline - time.monotonic()
if remaining <= 0:
- self.queue.remove(ticket)
+ self.queue.remove(entry)
self.timed_out += 1
self.condition.notify_all()
raise APIError(429, "Timed out waiting for the inference engine.", None,
"queue_timeout", "rate_limit_error", {"Retry-After": "1"})
self.condition.wait(min(remaining, 0.25))
- self.queue.popleft()
+ self.queue.remove(entry)
self.free_slots.remove(available)
self.active += 1
self.admitted += 1
@@ -715,14 +731,40 @@ class APIHandler(BaseHTTPRequestHandler):
if "*" not in self.server.cors_origins:
self.send_header("Vary", "Origin")
+ LOOPBACK_HOSTS = {"127.0.0.1", "localhost", "::1", ""}
+
+ def _is_authed(self):
+ """True if no key is configured, or a correct Bearer key was presented."""
+ if not self.server.api_key:
+ return True
+ import hmac
+ provided = self.headers.get("Authorization", "")
+ return hmac.compare_digest(provided, f"Bearer {self.server.api_key}")
+
def require_auth(self):
- if self.server.api_key:
- import hmac
- provided = self.headers.get("Authorization", "")
- expected = f"Bearer {self.server.api_key}"
- if not hmac.compare_digest(provided, expected):
- raise APIError(401, "Invalid or missing API key.", None, "invalid_api_key",
- "authentication_error")
+ if not self._is_authed():
+ raise APIError(401, "Invalid or missing API key.", None, "invalid_api_key",
+ "authentication_error")
+
+ def _check_host(self):
+ """DNS-rebinding guard: a web page can resolve a hostname to 127.0.0.1 and
+ drive this local server unless we pin the Host header to loopback / the bind
+ address. Rejects requests whose Host is anything else. (#SEC-7)"""
+ host = self.headers.get("Host", "")
+ if host.startswith("["):
+ name = host[1:].split("]", 1)[0] # [ipv6]:port
+ elif host.count(":") == 1:
+ name = host.rsplit(":", 1)[0] # host:port / ipv4:port
+ else:
+ name = host # bare host / bracketless ipv6
+ name = name.strip().lower()
+ allowed = set(self.LOOPBACK_HOSTS)
+ try:
+ allowed.add(str(self.server.server_address[0]).strip("[]").lower())
+ except Exception:
+ pass
+ if name not in allowed:
+ raise APIError(403, "Host header not allowed.", None, "forbidden")
def read_json(self):
try:
@@ -780,20 +822,26 @@ class APIHandler(BaseHTTPRequestHandler):
def do_GET(self):
request_id = "req_" + uuid.uuid4().hex
try:
+ self._check_host()
path = urlsplit(self.path).path
if path == "/health":
- payload = {"status": "ok", "scheduler": self.server.scheduler.snapshot(),
- "kv_slots": self.server.kv_slots}
- tiers = getattr(self.server.engine, "tiers", None) if self.server.engine else None
- if tiers: payload["tiers"] = tiers
- hwinfo = getattr(self.server.engine, "hwinfo", None) if self.server.engine else None
- if hwinfo: payload["hwinfo"] = hwinfo
+ # Liveness is always public; hardware/scheduler internals only when a
+ # request is authed (or no key set), so a configured key isn't leaked
+ # past a bare 200 to an unauthenticated probe. (#SEC-8)
+ payload = {"status": "ok"}
+ if self._is_authed():
+ payload["scheduler"] = self.server.scheduler.snapshot()
+ payload["kv_slots"] = self.server.kv_slots
+ tiers = getattr(self.server.engine, "tiers", None) if self.server.engine else None
+ if tiers: payload["tiers"] = tiers
+ hwinfo = getattr(self.server.engine, "hwinfo", None) if self.server.engine else None
+ if hwinfo: payload["hwinfo"] = hwinfo
self.send_json(200, payload, request_id)
return
if path == "/experts":
- eng = self.server.engine
payload = {"rows": 0, "cols": 0, "map": "", "hits": "", "seq": 0}
- if eng and getattr(eng, "emap", None):
+ eng = self.server.engine
+ if self._is_authed() and eng and getattr(eng, "emap", None): # (#SEC-8) hide routing telemetry unless authed
payload.update(eng.emap)
payload["hits"] = eng.hits or ""
payload["seq"] = eng.hits_seq
@@ -819,6 +867,13 @@ class APIHandler(BaseHTTPRequestHandler):
self.send_json(error.status, error_object(error), request_id, error.headers)
def do_OPTIONS(self):
+ try: # (#SEC-7) apply the Host guard uniformly, incl. CORS preflight
+ self._check_host()
+ except APIError:
+ self.send_response(403)
+ self.send_header("Content-Length", "0")
+ self.end_headers()
+ return
self.send_response(204)
self.send_header("Content-Length", "0")
self.send_cors_headers()
@@ -827,6 +882,7 @@ class APIHandler(BaseHTTPRequestHandler):
def do_POST(self):
request_id = "req_" + uuid.uuid4().hex
try:
+ self._check_host()
self.require_auth()
body = self.read_json()
self.check_model(body)
@@ -1027,11 +1083,12 @@ class APIHandler(BaseHTTPRequestHandler):
if include_usage:
event([], self.usage(stats))
if connected:
- try:
- self.wfile.write(b"data: [DONE]\n\n")
- self.wfile.flush()
- except OSError:
- pass
+ with ka_lock: # (#B9) share the pump's lock so [DONE] can't interleave a keepalive write
+ try:
+ self.wfile.write(b"data: [DONE]\n\n")
+ self.wfile.flush()
+ except OSError:
+ pass
self.close_connection = True
def client_disconnected(self):
@@ -1094,7 +1151,15 @@ def serve(model, host="127.0.0.1", port=8000, model_id="glm-5.2-colibri", api_ke
if not 1 <= kv_slots <= 16:
raise ValueError("kv_slots must be between 1 and 16")
if host not in ("127.0.0.1", "localhost", "::1") and not api_key:
- print("WARNING: API is listening beyond localhost without COLI_API_KEY", file=sys.stderr)
+ # (#SEC-6) Fail closed: an unauthenticated engine on a non-loopback bind exposes
+ # a compute-heavy API to the network. Refuse unless explicitly overridden.
+ if os.environ.get("COLI_ALLOW_INSECURE_BIND") == "1":
+ print("WARNING: binding %s beyond localhost with NO auth (COLI_ALLOW_INSECURE_BIND=1)" % host,
+ file=sys.stderr)
+ else:
+ print("refusing to bind %s beyond localhost without COLI_API_KEY set "
+ "(set COLI_ALLOW_INSECURE_BIND=1 to override)" % host, file=sys.stderr)
+ sys.exit(1)
origins = DEFAULT_CORS_ORIGINS if cors_origins is None else tuple(cors_origins)
# Bind before starting the 744B engine. A stale/occupied port must fail in
# milliseconds rather than loading hundreds of GB and leaking a child.
diff --git a/c/st.h b/c/st.h
index efa6e5a0..67b48aa8 100644
--- a/c/st.h
+++ b/c/st.h
@@ -200,11 +200,26 @@ static void st_init(shards *S, const char *snap_dir) {
if (a0 < 0 || b0 < a0 || data_start + b0 > fsz) {
fprintf(stderr, "%s: tensor '%s' data_offsets [%lld,%lld] out of file bounds (%lld)\n",
files[fi], name, (long long)a0, (long long)b0, (long long)fsz); exit(1); }
- int64_t numel = 1; for (int k = 0; k < shp->len; k++) numel *= (int64_t)shp->kids[k]->num;
+ int64_t numel = 1;
+ for (int k = 0; k < shp->len; k++) {
+ int64_t dim = (int64_t)shp->kids[k]->num;
+ if (dim < 0 || (dim && numel > INT64_MAX / dim)) { /* shape-product overflow from a crafted header */
+ fprintf(stderr, "%s: tensor '%s' shape overflows int64\n", files[fi], name); exit(1); }
+ numel *= dim;
+ }
if (S->n == S->cap) { S->cap *= 2; S->t = realloc(S->t, S->cap*sizeof(st_tensor)); }
st_tensor *t = &S->t[S->n++];
t->name = strdup(name); t->fd = fd; t->off = data_start + a0;
t->nbytes = b0 - a0; t->dtype = st_dtype_code(dt->str); t->numel = numel;
+ /* cross-check the declared element count against the byte span for FLOAT
+ * dtypes: st_read_f32 writes `numel` floats (BF16/F16 loop or F32 memcpy)
+ * into a caller-sized buffer, so a header with numel != nbytes/esz is an
+ * OOB write primitive. U8/I8 (raw quant bytes) are read by byte count, so
+ * their numel is unused by the read path and legitimately may differ. */
+ { int esz = t->dtype==2 ? 4 : (t->dtype==3 ? 1 : 2);
+ if (t->dtype != 3 && t->nbytes != numel * (int64_t)esz) {
+ fprintf(stderr, "%s: tensor '%s' numel %lld disagrees with byte span %lld (esz %d)\n",
+ files[fi], name, (long long)numel, (long long)t->nbytes, esz); exit(1); } }
}
free(arena); /* i jval restano leakati: ok, una tantum all'avvio */
free(hdr);
@@ -264,6 +279,20 @@ static int64_t st_read_f32(shards *S, const char *name, float *out, int drop) {
return t->numel;
}
+/* like st_read_f32 but refuses to write more than `cap` floats into `out`.
+ * Callers that size `out` from CONFIG dims (qt_alloc's O*I, per-row scales) must
+ * use this: a crafted header whose tensor holds more elements than the config
+ * shape would otherwise overrun `out`. Callers that size `out` from st_numel are
+ * self-consistent and may keep using st_read_f32. */
+static int64_t st_read_f32_cap(shards *S, const char *name, float *out, int64_t cap, int drop) {
+ st_tensor *t = st_find(S, name);
+ if (!t) { fprintf(stderr, "missing tensor: %s\n", name); exit(1); }
+ if (t->numel > cap) {
+ fprintf(stderr, "tensor %s: numel %lld exceeds destination capacity %lld\n",
+ name, (long long)t->numel, (long long)cap); exit(1); }
+ return st_read_f32(S, name, out, drop);
+}
+
static int64_t st_numel(shards *S, const char *name) {
st_tensor *t = st_find(S, name); return t ? t->numel : -1;
}
@@ -287,9 +316,13 @@ static void st_read_slice_f32(shards *S, const char *name, int64_t elem_off, int
st_tensor *t = st_find(S, name);
if (!t) { fprintf(stderr, "missing tensor: %s\n", name); exit(1); }
int esz = (t->dtype == 2) ? 4 : 2;
+ if (elem_off < 0 || n_elems < 0 || elem_off + n_elems > t->numel) { /* keep the slice inside the tensor */
+ fprintf(stderr, "slice %s [%lld,+%lld) out of tensor bounds (numel %lld)\n",
+ name, (long long)elem_off, (long long)n_elems, (long long)t->numel); exit(1); }
int64_t boff = t->off + elem_off * esz, nb = n_elems * esz;
void *raw = malloc(nb);
- st_pread_full(t->fd, raw, nb, boff, "pread slice");
+ if (!raw) { fprintf(stderr, "malloc %lld bytes for slice %s failed\n", (long long)nb, name); exit(1); }
+ st_pread_full(t->fd, raw, nb, boff, "pread slice"); /* dev #331: chunked + EINTR + honest short-read */
if (t->dtype == 2) memcpy(out, raw, nb);
else if (t->dtype == 0) { uint16_t *p = raw; for (int64_t i = 0; i < n_elems; i++) out[i] = bf16_to_f32(p[i]); }
else { uint16_t *p = raw; for (int64_t i = 0; i < n_elems; i++) out[i] = f16_to_f32(p[i]); }
diff --git a/c/tok.h b/c/tok.h
index 7d1140bb..df8e59c2 100644
--- a/c/tok.h
+++ b/c/tok.h
@@ -88,8 +88,14 @@ static void tk_build_bytemap(Tok *T){
/* ---------- caricamento tokenizer.json ---------- */
static char *tk_read_file(const char *path, long *out_n){
FILE *f=fopen(path,"rb"); if(!f){ perror(path); exit(1); }
- fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET);
- char *b=malloc(n+1); if(fread(b,1,n,f)!=(size_t)n){} b[n]=0; fclose(f); if(out_n)*out_n=n; return b;
+ if(fseek(f,0,SEEK_END)!=0){ perror(path); exit(1); }
+ long n=ftell(f);
+ if(n<0){ perror(path); exit(1); }
+ if(n>(1L<<30)){ fprintf(stderr,"%s: file too large (%ld bytes)\n",path,n); exit(1); } /* sanity cap vs a hostile size */
+ if(fseek(f,0,SEEK_SET)!=0){ perror(path); exit(1); }
+ char *b=malloc((size_t)n+1); if(!b){ fprintf(stderr,"OOM reading %s (%ld bytes)\n",path,n); exit(1); }
+ if(fread(b,1,(size_t)n,f)!=(size_t)n){ fprintf(stderr,"%s: short read\n",path); exit(1); }
+ b[n]=0; fclose(f); if(out_n)*out_n=n; return b;
}
static int cmp_sp_len(const void *a, const void *b){ return ((const Special*)b)->len - ((const Special*)a)->len; }
@@ -104,10 +110,19 @@ static void tok_load(Tok *T, const char *path){
jval *added=json_get(root,"added_tokens");
if(!vocab||!merges){ fprintf(stderr,"tokenizer.json: missing model.vocab/merges\n"); exit(1); }
- /* id massimo per dimensionare id2str */
+ /* id massimo per dimensionare id2str. Gli id vengono da un tokenizer.json di
+ * mirror non fidato: un id NEGATIVO indicizzerebbe id2str[id] SOTTO l'allocazione
+ * (OOB write) e un added_token privo di "id"/"content" darebbe NULL-deref. */
int maxid=0;
- for(int i=0;i<vocab->len;i++){ int id=(int)vocab->kids[i]->num; if(id>maxid)maxid=id; }
- if(added) for(int i=0;i<added->len;i++){ int id=(int)json_get(added->kids[i],"id")->num; if(id>maxid)maxid=id; }
+ for(int i=0;i<vocab->len;i++){ int id=(int)vocab->kids[i]->num;
+ if(id<0){ fprintf(stderr,"tokenizer.json: negative vocab id %d\n",id); exit(1); }
+ if(id>maxid)maxid=id; }
+ if(added) for(int i=0;i<added->len;i++){
+ jval *ji=json_get(added->kids[i],"id");
+ if(!ji||ji->t!=J_NUM){ fprintf(stderr,"tokenizer.json: added_token missing numeric id\n"); exit(1); }
+ int id=(int)ji->num;
+ if(id<0){ fprintf(stderr,"tokenizer.json: negative added id %d\n",id); exit(1); }
+ if(id>maxid)maxid=id; }
T->n_ids=maxid+1;
T->id2str=calloc(T->n_ids,sizeof(char*));
T->id_added=calloc(T->n_ids,sizeof(int));
@@ -136,7 +151,10 @@ static void tok_load(Tok *T, const char *path){
T->nsp=added->len; T->sp=calloc(T->nsp,sizeof(Special));
for(int i=0;i<added->len;i++){
jval *a=added->kids[i];
- char *content=json_get(a,"content")->str; int id=(int)json_get(a,"id")->num;
+ jval *jc=json_get(a,"content"), *ji=json_get(a,"id");
+ if(!jc||jc->t!=J_STR||!jc->str||!ji||ji->t!=J_NUM){ fprintf(stderr,"tokenizer.json: malformed added_token\n"); exit(1); }
+ char *content=jc->str; int id=(int)ji->num;
+ if(id<0||id>maxid){ fprintf(stderr,"tokenizer.json: added id %d out of range\n",id); exit(1); }
T->sp[i].str=content; T->sp[i].len=(int)strlen(content); T->sp[i].id=id;
T->id2str[id]=content; T->id_added[id]=1;
jval *sf=json_get(a,"special"); /* "special": true/false */
diff --git a/c/tools/download_glm52.py b/c/tools/download_glm52.py
index aa0c791a..ebd3b2fe 100644
--- a/c/tools/download_glm52.py
+++ b/c/tools/download_glm52.py
@@ -18,12 +18,15 @@ import os, sys, shutil
from huggingface_hub import snapshot_download, HfApi
REPO = "zai-org/GLM-5.2-FP8"
+# Pin the model revision for supply-chain integrity: set GLM_REVISION to a commit SHA
+# so a mutated/compromised upstream can't silently swap the weights you fetch.
+REVISION = os.environ.get("GLM_REVISION", "main")
DEST = os.environ.get("GLM_DIR", "/home/vincenzo/glm52") # su ext4 (/dev/sdd), MAI su /mnt/c
def human(n): return f"{n/1e9:.0f} GB"
def check():
- info = HfApi().repo_info(REPO, files_metadata=True)
+ info = HfApi().repo_info(REPO, revision=REVISION, files_metadata=True)
tot = sum((s.size or 0) for s in info.siblings)
sts = [s for s in info.siblings if s.rfilename.endswith(".safetensors")]
free = shutil.disk_usage(os.path.dirname(DEST) or "/").free
@@ -40,6 +43,7 @@ def download():
# resume_download e' implicito; in caso di interruzione, rilancia e riprende.
snapshot_download(
repo_id=REPO,
+ revision=REVISION,
local_dir=DEST,
allow_patterns=["*.safetensors", "*.json", "*.txt", "*.model"],
max_workers=8,
diff --git a/c/tools/requirements.txt b/c/tools/requirements.txt
new file mode 100644
index 00000000..3d0087c6
--- /dev/null
+++ b/c/tools/requirements.txt
@@ -0,0 +1,15 @@
+# Python dependencies for the colibri offline converter / oracle / download tools
+# (c/tools/*.py and c/*.py). Minimum-version pins; for a fully reproducible env,
+# freeze exact versions into your own lockfile with `pip freeze > locked.txt`.
+torch>=2.4
+safetensors>=0.4
+huggingface-hub>=0.24
+numpy>=1.26
+tokenizers>=0.20
+transformers>=5.11 # <5.11 applies split-half Llama RoPE and drifts vs the engine's interleaved MLA (issue #281)
+datasets>=2.20
+requests>=2.31
+
+# Optional download-fallback backends (only needed for download_fp8.py's alt paths):
+# modelscope>=1.9
+# hf_transfer>=0.1
diff --git a/flake.nix b/flake.nix
index 584a0886..c1b90792 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,6 +1,9 @@
{
description = "colibrì — run GLM-5.2 (744B MoE) on a consumer machine with ~25 GB RAM";
+ # Reproducibility: these inputs track a branch, so torch/numpy/etc. float across
+ # rebuilds. For deterministic builds run `nix flake lock` once and COMMIT the
+ # generated flake.lock (it pins each input to a commit SHA). (#D2)
inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/nixos-26.05";
flake-utils.url = "github:numtide/flake-utils";
diff --git a/web/src/App.tsx b/web/src/App.tsx
index a33cc880..3f86f74f 100644
--- a/web/src/App.tsx
+++ b/web/src/App.tsx
@@ -161,7 +161,8 @@ export default function App() {
} catch (cause) {
if (controller.signal.aborted) return
setConnected(false)
- setError(cause instanceof Error ? cause.message : "Could not reach the server.")
+ const msg = cause instanceof Error ? cause.message : "Could not reach the server."
+ setError(/401|unauthor/i.test(msg) ? "Authentication failed — check the API key." : msg)
} finally {
if (probeRef.current === controller) { probeRef.current = null; setConnecting(false) }
}
@@ -247,12 +248,12 @@ export default function App() {
<section className="side-section">
<div className="section-title"><Link2 className="size-3.5" /> Connection</div>
<label>API endpoint<Input value={baseUrl} onChange={(event) => setBaseUrl(event.target.value)} /></label>
- <label>API key<div className="relative"><KeyRound className="field-icon" /><Input className="pl-9" type="password" value={apiKey} placeholder="optional" onChange={(event) => setApiKey(event.target.value)} /></div><span className="field-help">Kept in memory only · sent to this endpoint</span></label>
+ <label>API key<div className="relative"><KeyRound className="field-icon" /><Input className="pl-9" type="password" autoComplete="new-password" value={apiKey} placeholder="optional" onChange={(event) => setApiKey(event.target.value)} /></div><span className="field-help">Kept in memory only · sent to this endpoint</span></label>
<Button type="button" variant="secondary" onClick={connect} disabled={connecting}>
{connecting ? <LoaderCircle className="size-4 animate-spin" /> : <RefreshCw className="size-4" />}
Probe server
</Button>
- <div className={cn("connection-state", connected && "connected")} aria-live="polite"><span />{connected ? "Engine reachable" : "Not connected"}</div>
+ <div className={cn("connection-state", connected && "connected")} aria-live="polite"><span />{connected ? "Engine reachable" : connecting ? "Probing…" : "Not connected"}</div>
</section>
<section className="side-section runtime-section" aria-live="polite">
@@ -339,7 +340,7 @@ export default function App() {
<h2>Ask the giant.<br /><em>Keep the machine yours.</em></h2>
<p>Connect to a local colibrì server and stream responses directly from your hardware. Nothing leaves the endpoint you choose.</p>
<div className="suggestions">
- {["Explain how expert routing works", "Write a small C benchmark", "Compare RAM and VRAM caching"].map((item) => <button key={item} onClick={() => setDraft(item)}>{item}<ArrowUp className="size-3.5 rotate-45" /></button>)}
+ {["Explain how expert routing works", "Write a small C benchmark", "Compare RAM and VRAM caching"].map((item) => <button type="button" key={item} onClick={() => setDraft(item)}>{item}<ArrowUp className="size-3.5 rotate-45" /></button>)}
</div>
</div>
) : (
diff --git a/web/src/Brain.tsx b/web/src/Brain.tsx
index 6811238f..b5635e74 100644
--- a/web/src/Brain.tsx
+++ b/web/src/Brain.tsx
@@ -27,6 +27,7 @@ export function Brain({ baseUrl, apiKey, connected }: { baseUrl: string; apiKey:
const wrapRef = useRef<HTMLDivElement>(null)
const [wrapSize, setWrapSize] = useState({ w: 1200, h: 700 })
const [data, setData] = useState<ExpertMap | null>(null)
+ const [probeErr, setProbeErr] = useState(false) // (#U3) surface /experts failures instead of an endless spinner
const [atlas, setAtlas] = useState<Record<string, AtlasEntry> | null>(null)
const [tip, setTip] = useState<{ x: number; y: number; row: number; col: number; tier: number; heat: number } | null>(null)
const pulseRef = useRef<Float32Array | null>(null) // per-expert pulse intensity 0..1
@@ -59,9 +60,11 @@ export function Brain({ baseUrl, apiKey, connected }: { baseUrl: string; apiKey:
const poll = async () => {
try {
const res = await fetch(endpoint(base, "/experts"), { headers: apiKey ? { Authorization: `Bearer ${apiKey}` } : {} })
+ if (!res.ok) throw new Error(`/experts ${res.status}`)
const next = (await res.json()) as ExpertMap
if (disposed || !next.rows) return
setData(next)
+ setProbeErr(false)
if (next.seq !== lastSeq.current && next.hits) {
lastSeq.current = next.seq
const n = next.rows * next.cols
@@ -72,7 +75,7 @@ export function Brain({ baseUrl, apiKey, connected }: { baseUrl: string; apiKey:
if (byte & (1 << (i & 7))) p[i] = 1
}
}
- } catch { /* engine busy or restarting — keep the last frame */ }
+ } catch { if (!disposed) setProbeErr(true) /* surface repeated failures; keep the last frame */ }
}
void poll()
const t = window.setInterval(() => void poll(), 1500)
@@ -154,13 +157,14 @@ export function Brain({ baseUrl, apiKey, connected }: { baseUrl: string; apiKey:
<div className="brain-canvas-wrap" ref={wrapRef}>
<canvas ref={canvasRef} onMouseMove={onMove} onMouseLeave={() => setTip(null)} />
{!connected && <p className="runtime-unavailable">Connect to the engine to see the cortex.</p>}
+ {connected && probeErr && !data && <p className="runtime-unavailable">Expert map unavailable — this engine build may not expose /experts.</p>}
</div>
{tip && data && (() => {
const isMtp = tip.row === data.rows - 1
const realLayer = isMtp ? 78 : tip.row + 3
const entry = atlas?.[`${realLayer}:${tip.col}`]
return (
- <div className="brain-tip" style={{ left: tip.x + 14, top: tip.y + 14 }}>
+ <div className="brain-tip" style={{ left: Math.min(tip.x + 14, window.innerWidth - 260), top: Math.min(tip.y + 14, window.innerHeight - 170) }}>
<div className="brain-tip-title"><Layers className="size-3" /> Layer {realLayer}{isMtp ? " (MTP)" : ""} · Expert {tip.col}</div>
<div>Tier: <strong style={{ color: ["#8b9aa3", "#5a9bd8", "#4ed6a5"][tip.tier] }}>{TIER_NAME[tip.tier]}</strong></div>
<div>Heat: <strong>{tip.heat === 0 ? "never routed" : `~2^${tip.heat} selections`}</strong></div>
diff --git a/web/src/ErrorBoundary.tsx b/web/src/ErrorBoundary.tsx
index f88b8a98..40df0647 100644
--- a/web/src/ErrorBoundary.tsx
+++ b/web/src/ErrorBoundary.tsx
@@ -9,11 +9,11 @@ export class ErrorBoundary extends Component<{ children: ReactNode }, State> {
}
render() {
if (!this.state.error) return this.props.children
- return <div style={{ padding: "2rem", fontFamily: "ui-monospace, monospace", color: "#e5e7eb", background: "#0b0f10", minHeight: "100vh" }}>
- <h2 style={{ color: "#4ed6a5" }}>colibrì UI hit an error</h2>
- <p style={{ color: "#9ca3af" }}>The engine is unaffected. Try refreshing.</p>
- <pre style={{ whiteSpace: "pre-wrap", color: "#f87171" }}>{String(this.state.error)}</pre>
- <button onClick={() => this.setState({ error: null, stack: "" })} style={{ marginTop: "1rem", padding: "0.5rem 1rem", background: "#1f2937", color: "#e5e7eb", border: "1px solid #374151", borderRadius: 8, cursor: "pointer" }}>Retry</button>
+ return <div style={{ padding: "2rem", fontFamily: "ui-monospace, monospace", color: "var(--foreground)", background: "var(--background)", minHeight: "100vh" }}>
+ <h2 style={{ color: "var(--primary)" }}>colibrì UI hit an error</h2>
+ <p style={{ color: "var(--muted-foreground)" }}>The engine is unaffected. Try refreshing.</p>
+ <pre style={{ whiteSpace: "pre-wrap", color: "var(--destructive)" }}>{String(this.state.error)}</pre>
+ <button onClick={() => this.setState({ error: null, stack: "" })} style={{ marginTop: "1rem", padding: "0.5rem 1rem", background: "var(--secondary)", color: "var(--foreground)", border: "1px solid var(--border)", borderRadius: 8, cursor: "pointer" }}>Retry</button>
</div>
}
}
diff --git a/web/src/index.css b/web/src/index.css
index 8d777f41..ee315817 100644
--- a/web/src/index.css
+++ b/web/src/index.css
@@ -69,7 +69,7 @@ button:focus-visible, input:focus-visible, textarea:focus-visible, select:focus-
.chat-panel { min-width: 0; height: 100vh; display: grid; grid-template-rows: 72px minmax(0, 1fr) auto; }
.topbar { display: flex; align-items: center; justify-content: space-between; padding: 0 32px; border-bottom: 1px solid var(--border); }
.topbar > div:first-child { min-width: 0; display: grid; gap: 3px; }.topbar strong { overflow: hidden; font: 500 14px ui-monospace, SFMono-Regular, monospace; text-overflow: ellipsis; white-space: nowrap; }.eyebrow { color: #718086; font-size: 9px; font-weight: 800; letter-spacing: .16em; }
-.top-actions { display: flex; align-items: center; gap: 8px; }.top-actions [data-slot=badge] { gap: 5px; }
+.top-actions { display: flex; flex-wrap: wrap; align-items: center; gap: 8px; }.top-actions [data-slot=badge] { gap: 5px; }
.conversation { overflow-y: auto; }
.empty-state { width: min(680px, calc(100% - 40px)); min-height: 100%; margin: 0 auto; display: flex; flex-direction: column; align-items: center; justify-content: center; padding: 56px 0; text-align: center; }
.orb { position: relative; display: grid; place-items: center; width: 66px; height: 66px; margin-bottom: 24px; border: 1px solid rgba(78,214,165,.25); border-radius: 50%; color: var(--primary); background: radial-gradient(circle, rgba(78,214,165,.17), rgba(78,214,165,.02) 68%); box-shadow: 0 0 55px rgba(78,214,165,.1); }.orb svg { width: 25px; }.orb::after { content: ""; position: absolute; inset: -7px; border: 1px dashed rgba(78,214,165,.14); border-radius: inherit; }