Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(glm53): correct RAM sizing and Apple unified-memory planning
trigger2k2058ed660b453f
diff --git a/c/resource_plan.py b/c/resource_plan.pyindex 7057b7fb..7a2fe3b0 100644--- a/c/resource_plan.py+++ b/c/resource_plan.py@@ -3,6 +3,7 @@import jsonimport os+import platformimport reimport shutilimport statistics@@ -330,6 +331,17 @@ def memory_available():return 0+def _host_unified_memory():+ """Whether the host itself exposes one CPU/GPU physical memory pool.++ This is intentionally separate from accelerator placement. A CPU-only+ engine such as glm53 still runs on unified-memory Apple Silicon, but that+ fact must not fabricate a VRAM tier or make an unrelated GPU steer cache+ placement.+ """+ return sys.platform == "darwin" and platform.machine().lower() in ("arm64", "aarch64")++# Strict .coli_ssd grammar -- the byte-for-byte mirror of colibri.c's# coli_ssd_cache_parse() (keep the two in lockstep; test_resource_plan.py and# test_ssd_probe.c chew the same vector file, tests/fixtures/ssd_cache_vectors.txt):@@ -795,12 +807,19 @@ def cpu_socket_count():return 1-def _auto_tune(bottleneck_class, projected_hit, gpus, cpu_sockets, plan_has_metal):+def _auto_tune(bottleneck_class, projected_hit, gpus, cpu_sockets, plan_has_metal,+ engine_group=None):"""Derive tuning knobs from the bottleneck classification."""tune = {}has_gpu = bool(gpus)n_gpu = len(gpus)+ # glm53 has its own loader/cache controls and does not consume the generic+ # DRAFT/PIPE/PIN/NUMA knobs below. Recommending them is worse than leaving+ # them unset because `coli tune` then reports changes the engine ignores.+ if engine_group == "glm53":+ return tune+# MTP: costs more than it saves when compute-bound (#389 measured 42% loss)# or streaming-bound (#467 measured 32% loss under CUDA at 85% hit).# EXCEPTION: an explicit COLI_CUDA_MTP=1 in the environment is a documented@@ -950,7 +969,9 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,# plans_placement().planning_gpus = [gpu for gpu in gpus if plans_placement(gpu)]- unified = any(gpu.get("unified_memory", False) for gpu in planning_gpus)+ placement_unified = any(gpu.get("unified_memory", False)+ for gpu in planning_gpus)+ unified = placement_unified or _host_unified_memory()typical = info["typical_expert_bytes"]max_expert = info["max_expert_bytes"] or typicalkv_bytes = (geometry.context_state_bytes + geometry.fixed_state_bytes) * kv_slots@@ -974,17 +995,17 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,requested_vram = int(vram_gb * GB) if vram_gb > 0 else safe_vramrequested_vram_before_clamp = requested_vramunified_pool = max(0, available_memory - info["dense_bytes"] - runtime_bytes)- if unified:+ if placement_unified:# Unified devices expose one physical pool to CUDA and the host. Do not# let an expert tier consume pages that the RAM tier also believes are# available. Dense/runtime reservations are shared exactly once below.requested_vram = min(requested_vram, unified_pool)- vram_limit = unified_pool if unified else safe_vram+ vram_limit = unified_pool if placement_unified else safe_vramvram_budget = min(requested_vram, vram_limit, info["expert_bytes"])vram_experts = int(vram_budget // typical) if typical else 0hot_bytes = min(info["expert_bytes"], vram_experts * typical)warnings = []- if unified:+ if placement_unified:requested_ram = int(ram_gb * GB) if ram_gb > 0 else int(available_memory * 0.88)requested_ram_experts = max(0, requested_ram - info["dense_bytes"] - runtime_bytes)ram_expert_bytes = min(requested_ram_experts,@@ -997,7 +1018,7 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,else:ram_budget = int(ram_gb * GB) if ram_gb > 0 else int(available_memory * 0.88)if ram_budget < 4 * GB:- ram_budget = 8 * GB if not unified else max(0, ram_budget)+ ram_budget = 8 * GB if not placement_unified else max(0, ram_budget)cache_bytes = max(0, ram_budget - info["dense_bytes"] - runtime_bytes)cap = int(cache_bytes // per_cap) if per_cap else 0if configured_experts:@@ -1017,7 +1038,7 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,f"GPU {gpu['index']} ({gpu['name']}) was detected but its free memory is ""not qualified as a placement budget on this platform; it is reported ""only and drives no automatic tier")- if unified:+ if placement_unified:warnings.append("GPU and RAM share one physical memory pool; budgets were jointly constrained")# The plan sizes the hot tier from *free* VRAM, so running it while an engine@@ -1058,7 +1079,8 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,bottleneck_class = "memory"tune = _auto_tune(bottleneck_class, projected_hit, planning_gpus, cpu_sockets,- plan_has_metal=False)+ plan_has_metal=False,+ engine_group=resolved.descriptor.engine_group)probe_state, probe_gbs = ssd_probe_state(info["path"])actions = _next_actions(bottleneck_class, projected_hit, probe_state,probe_gbs, planning_gpus)diff --git a/c/tests/test_resource_plan.py b/c/tests/test_resource_plan.pyindex ce63dfb8..ede6e88c 100644--- a/c/tests/test_resource_plan.py+++ b/c/tests/test_resource_plan.py@@ -97,6 +97,32 @@ class ResourcePlanTest(unittest.TestCase):# 0 slots/layer. The value must be a sane positive number of bytes.self.assertGreater(memory_available(), 0)+ def test_apple_silicon_reports_unified_host_memory_without_fake_vram(self):+ # Host memory topology is a hardware fact, independent of whether the+ # selected engine can place anything on the GPU. In particular glm53+ # is CPU-only today, but an M-series Mac must not be reported as+ # memory.unified=false merely because planning_gpus is empty.+ with mock.patch.object(sys, "platform", "darwin"), \+ mock.patch("resource_plan.platform.machine", return_value="arm64"):+ plan = build_plan(self.model, ram_gb=16, available_memory=32 * GB,+ available_disk=1, gpus=[], physical_cpus=8,+ cpu_sockets=1)+ self.assertTrue(plan["memory"]["unified"])+ self.assertEqual(plan["tiers"]["vram"]["budget_bytes"], 0)+ self.assertFalse(any("jointly constrained" in warning+ for warning in plan["warnings"]))++ def test_glm53_auto_tune_does_not_emit_generic_inert_knobs(self):+ from resource_plan import _auto_tune++ generic = _auto_tune("disk", 0.50, [], 1, False)+ self.assertIn("DRAFT", generic)+ self.assertIn("PIPE", generic)++ glm53 = _auto_tune("disk", 0.50, [], 1, False,+ engine_group="glm53")+ self.assertEqual(glm53, {})+def test_cpu_socket_count_is_positive(self):self.assertGreaterEqual(cpu_socket_count(), 1)diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.mdindex a87b1e0c..a2369741 100644--- a/docs/ENVIRONMENT.md+++ b/docs/ENVIRONMENT.md@@ -323,7 +323,7 @@ See `docs/glm53-flash.md`.| Variable | Default | Effect ||---|---|---|| `GLM53_BITS` | `4` | Precision of the resident dense weights: 4, 8 or 32. Routed experts are not affected — they arrive already quantized in the container and are never requantized. |-| `GLM53_EXPERT_GB` | measured | RAM budget (GB) for the expert LRU cache; per-layer slots are derived from it. Unset, it is taken from `MemAvailable` after the weights are loaded, minus a 3 GB margin. A fixed number is wrong in both directions: too small on a large machine leaves memory idle while the disk does all the work. |+| `GLM53_EXPERT_GB` | measured | RAM budget (GB) for the expert LRU cache; per-layer slots are derived from it. Unset, it is taken from reclaimable physical memory after the weights are loaded (Linux `MemAvailable`, Windows available physical memory, macOS free+inactive+purgeable pages), minus a 3 GB margin. A fixed number is wrong in both directions: too small on a large machine leaves memory idle while the disk does all the work. || `GLM53_MAXT` | `8192` | KV state capacity in tokens, and the session size in serve mode. || `GLM53_PREFILL_CHUNK` | `128` | Prefill chunk size in tokens. Smaller keeps the workspace smaller; too small re-reads experts once per chunk per layer instead of amortizing them. || `GLM53_MAX_IMAGE_TOKENS` | checkpoint's (8000) | Ceiling on tokens per image. Each covers 28×28 pixels, so 256 keeps ordinary text legible and 64 keeps shapes and colours. The image is shrunk, not cropped. Lower it: 8000 is 2691 tokens for a 1080p photo, i.e. a prefill nobody will sit through. |