Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(glm53): correct RAM sizing and Apple unified-memory planning

trigger2k2058ed660b453f
diff --git a/c/resource_plan.py b/c/resource_plan.py
index 7057b7fb..7a2fe3b0 100644
--- a/c/resource_plan.py
+++ b/c/resource_plan.py
@@ -3,6 +3,7 @@
import json
import os
+import platform
import re
import shutil
import statistics
@@ -330,6 +331,17 @@ def memory_available():
return 0
+def _host_unified_memory():
+ """Whether the host itself exposes one CPU/GPU physical memory pool.
+
+ This is intentionally separate from accelerator placement. A CPU-only
+ engine such as glm53 still runs on unified-memory Apple Silicon, but that
+ fact must not fabricate a VRAM tier or make an unrelated GPU steer cache
+ placement.
+ """
+ return sys.platform == "darwin" and platform.machine().lower() in ("arm64", "aarch64")
+
+
# Strict .coli_ssd grammar -- the byte-for-byte mirror of colibri.c's
# coli_ssd_cache_parse() (keep the two in lockstep; test_resource_plan.py and
# test_ssd_probe.c chew the same vector file, tests/fixtures/ssd_cache_vectors.txt):
@@ -795,12 +807,19 @@ def cpu_socket_count():
return 1
-def _auto_tune(bottleneck_class, projected_hit, gpus, cpu_sockets, plan_has_metal):
+def _auto_tune(bottleneck_class, projected_hit, gpus, cpu_sockets, plan_has_metal,
+ engine_group=None):
"""Derive tuning knobs from the bottleneck classification."""
tune = {}
has_gpu = bool(gpus)
n_gpu = len(gpus)
+ # glm53 has its own loader/cache controls and does not consume the generic
+ # DRAFT/PIPE/PIN/NUMA knobs below. Recommending them is worse than leaving
+ # them unset because `coli tune` then reports changes the engine ignores.
+ if engine_group == "glm53":
+ return tune
+
# MTP: costs more than it saves when compute-bound (#389 measured 42% loss)
# or streaming-bound (#467 measured 32% loss under CUDA at 85% hit).
# EXCEPTION: an explicit COLI_CUDA_MTP=1 in the environment is a documented
@@ -950,7 +969,9 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,
# plans_placement().
planning_gpus = [gpu for gpu in gpus if plans_placement(gpu)]
- unified = any(gpu.get("unified_memory", False) for gpu in planning_gpus)
+ placement_unified = any(gpu.get("unified_memory", False)
+ for gpu in planning_gpus)
+ unified = placement_unified or _host_unified_memory()
typical = info["typical_expert_bytes"]
max_expert = info["max_expert_bytes"] or typical
kv_bytes = (geometry.context_state_bytes + geometry.fixed_state_bytes) * kv_slots
@@ -974,17 +995,17 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,
requested_vram = int(vram_gb * GB) if vram_gb > 0 else safe_vram
requested_vram_before_clamp = requested_vram
unified_pool = max(0, available_memory - info["dense_bytes"] - runtime_bytes)
- if unified:
+ if placement_unified:
# Unified devices expose one physical pool to CUDA and the host. Do not
# let an expert tier consume pages that the RAM tier also believes are
# available. Dense/runtime reservations are shared exactly once below.
requested_vram = min(requested_vram, unified_pool)
- vram_limit = unified_pool if unified else safe_vram
+ vram_limit = unified_pool if placement_unified else safe_vram
vram_budget = min(requested_vram, vram_limit, info["expert_bytes"])
vram_experts = int(vram_budget // typical) if typical else 0
hot_bytes = min(info["expert_bytes"], vram_experts * typical)
warnings = []
- if unified:
+ if placement_unified:
requested_ram = int(ram_gb * GB) if ram_gb > 0 else int(available_memory * 0.88)
requested_ram_experts = max(0, requested_ram - info["dense_bytes"] - runtime_bytes)
ram_expert_bytes = min(requested_ram_experts,
@@ -997,7 +1018,7 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,
else:
ram_budget = int(ram_gb * GB) if ram_gb > 0 else int(available_memory * 0.88)
if ram_budget < 4 * GB:
- ram_budget = 8 * GB if not unified else max(0, ram_budget)
+ ram_budget = 8 * GB if not placement_unified else max(0, ram_budget)
cache_bytes = max(0, ram_budget - info["dense_bytes"] - runtime_bytes)
cap = int(cache_bytes // per_cap) if per_cap else 0
if configured_experts:
@@ -1017,7 +1038,7 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,
f"GPU {gpu['index']} ({gpu['name']}) was detected but its free memory is "
"not qualified as a placement budget on this platform; it is reported "
"only and drives no automatic tier")
- if unified:
+ if placement_unified:
warnings.append(
"GPU and RAM share one physical memory pool; budgets were jointly constrained")
# The plan sizes the hot tier from *free* VRAM, so running it while an engine
@@ -1058,7 +1079,8 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,
bottleneck_class = "memory"
tune = _auto_tune(bottleneck_class, projected_hit, planning_gpus, cpu_sockets,
- plan_has_metal=False)
+ plan_has_metal=False,
+ engine_group=resolved.descriptor.engine_group)
probe_state, probe_gbs = ssd_probe_state(info["path"])
actions = _next_actions(bottleneck_class, projected_hit, probe_state,
probe_gbs, planning_gpus)
diff --git a/c/tests/test_resource_plan.py b/c/tests/test_resource_plan.py
index ce63dfb8..ede6e88c 100644
--- a/c/tests/test_resource_plan.py
+++ b/c/tests/test_resource_plan.py
@@ -97,6 +97,32 @@ class ResourcePlanTest(unittest.TestCase):
# 0 slots/layer. The value must be a sane positive number of bytes.
self.assertGreater(memory_available(), 0)
+ def test_apple_silicon_reports_unified_host_memory_without_fake_vram(self):
+ # Host memory topology is a hardware fact, independent of whether the
+ # selected engine can place anything on the GPU. In particular glm53
+ # is CPU-only today, but an M-series Mac must not be reported as
+ # memory.unified=false merely because planning_gpus is empty.
+ with mock.patch.object(sys, "platform", "darwin"), \
+ mock.patch("resource_plan.platform.machine", return_value="arm64"):
+ plan = build_plan(self.model, ram_gb=16, available_memory=32 * GB,
+ available_disk=1, gpus=[], physical_cpus=8,
+ cpu_sockets=1)
+ self.assertTrue(plan["memory"]["unified"])
+ self.assertEqual(plan["tiers"]["vram"]["budget_bytes"], 0)
+ self.assertFalse(any("jointly constrained" in warning
+ for warning in plan["warnings"]))
+
+ def test_glm53_auto_tune_does_not_emit_generic_inert_knobs(self):
+ from resource_plan import _auto_tune
+
+ generic = _auto_tune("disk", 0.50, [], 1, False)
+ self.assertIn("DRAFT", generic)
+ self.assertIn("PIPE", generic)
+
+ glm53 = _auto_tune("disk", 0.50, [], 1, False,
+ engine_group="glm53")
+ self.assertEqual(glm53, {})
+
def test_cpu_socket_count_is_positive(self):
self.assertGreaterEqual(cpu_socket_count(), 1)
diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md
index a87b1e0c..a2369741 100644
--- a/docs/ENVIRONMENT.md
+++ b/docs/ENVIRONMENT.md
@@ -323,7 +323,7 @@ See `docs/glm53-flash.md`.
| Variable | Default | Effect |
|---|---|---|
| `GLM53_BITS` | `4` | Precision of the resident dense weights: 4, 8 or 32. Routed experts are not affected — they arrive already quantized in the container and are never requantized. |
-| `GLM53_EXPERT_GB` | measured | RAM budget (GB) for the expert LRU cache; per-layer slots are derived from it. Unset, it is taken from `MemAvailable` after the weights are loaded, minus a 3 GB margin. A fixed number is wrong in both directions: too small on a large machine leaves memory idle while the disk does all the work. |
+| `GLM53_EXPERT_GB` | measured | RAM budget (GB) for the expert LRU cache; per-layer slots are derived from it. Unset, it is taken from reclaimable physical memory after the weights are loaded (Linux `MemAvailable`, Windows available physical memory, macOS free+inactive+purgeable pages), minus a 3 GB margin. A fixed number is wrong in both directions: too small on a large machine leaves memory idle while the disk does all the work. |
| `GLM53_MAXT` | `8192` | KV state capacity in tokens, and the session size in serve mode. |
| `GLM53_PREFILL_CHUNK` | `128` | Prefill chunk size in tokens. Smaller keeps the workspace smaller; too small re-reads experts once per chunk per layer instead of amortizing them. |
| `GLM53_MAX_IMAGE_TOKENS` | checkpoint's (8000) | Ceiling on tokens per image. Each covers 28×28 pixels, so 256 keeps ordinary text legible and 64 keeps shapes and colours. The image is shrunk, not cropped. Lower it: 8000 is 2691 tokens for a 1080p photo, i.e. a prefill nobody will sit through. |