Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

experiment: add offline residency simulator

Mario Gutierrezcc18bfbb3f44
diff --git a/c/tests/test_residency_sim.py b/c/tests/test_residency_sim.py
new file mode 100644
index 00000000..4967e729
--- /dev/null
+++ b/c/tests/test_residency_sim.py
@@ -0,0 +1,351 @@
+import json
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+
+TOOLS = Path(__file__).resolve().parent.parent / "tools"
+sys.path.insert(0, str(TOOLS))
+import residency_sim as sim
+
+
+class TraceParserTest(unittest.TestCase):
+ def test_batch_rows_keep_union_order_and_selection_multiplicity(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ path = Path(tmp) / "trace.txt"
+ path.write_text(
+ "0 0 3 7:0.6000 2:0.4000\n"
+ "0 1 3 2:0.7000 9:0.3000\n"
+ "1 0 4 1:1.0000\n",
+ encoding="utf-8",
+ )
+ events = sim.parse_trace(path)
+ self.assertEqual(events, [
+ sim.AccessEvent(3, (7, 2, 9), (7, 2, 2, 9), 0, str(path)),
+ sim.AccessEvent(4, (1,), (1,), 1, str(path)),
+ ])
+
+ def test_multiple_calls_to_same_layer_remain_separate_events(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ path = Path(tmp) / "trace.txt"
+ path.write_text("0 0 3 1:1.0000\n1 0 3 2:1.0000\n", encoding="utf-8")
+ events = sim.parse_trace(path)
+ self.assertEqual([event.experts for event in events], [(1,), (2,)])
+
+ def test_noncontiguous_group_is_rejected(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ path = Path(tmp) / "trace.txt"
+ path.write_text(
+ "0 0 3 1:1.0000\n1 0 4 2:1.0000\n0 1 3 3:1.0000\n",
+ encoding="utf-8",
+ )
+ with self.assertRaisesRegex(ValueError, "noncontiguous"):
+ sim.parse_trace(path)
+
+ def test_kimi_style_constant_call_ids_are_rejected(self):
+ events = [
+ sim.AccessEvent(0, (1,), call=0),
+ sim.AccessEvent(1, (2,), call=0),
+ sim.AccessEvent(2, (3,), call=0),
+ ]
+ with self.assertRaisesRegex(ValueError, "lacks advancing GLM call ids"):
+ sim.validate_glm_trace(events, Path("kimi.trace"))
+
+ def test_glm_call_ids_advance_across_layer_wraps(self):
+ events = [
+ sim.AccessEvent(0, (1,), call=0),
+ sim.AccessEvent(1, (2,), call=1),
+ sim.AccessEvent(2, (3,), call=2),
+ sim.AccessEvent(0, (4,), call=3),
+ sim.AccessEvent(1, (5,), call=4),
+ ]
+ sim.validate_glm_trace(events, Path("glm.trace"))
+
+ def test_glm_call_ids_must_start_at_zero_and_advance_at_wrap(self):
+ invalid = (
+ [sim.AccessEvent(0, (1,), call=1)],
+ [sim.AccessEvent(0, (1,), call=0), sim.AccessEvent(1, (2,), call=1),
+ sim.AccessEvent(0, (3,), call=1)],
+ )
+ for events in invalid:
+ with self.subTest(events=events), self.assertRaisesRegex(
+ ValueError, "lacks advancing GLM call ids"):
+ sim.validate_glm_trace(events, Path("glm.trace"))
+
+ def test_rows_must_start_at_zero_and_be_contiguous(self):
+ invalid = (
+ "0 1 3 7:1.0000\n",
+ "0 0 3 7:1.0000\n0 2 3 8:1.0000\n",
+ )
+ for text in invalid:
+ with self.subTest(text=text), tempfile.TemporaryDirectory() as tmp:
+ path = Path(tmp) / "trace.txt"
+ path.write_text(text, encoding="utf-8")
+ with self.assertRaisesRegex(ValueError, "expected row"):
+ sim.parse_trace(path)
+
+ def test_malformed_or_nonfinite_gate_is_rejected_with_location(self):
+ invalid = ("0 0 3 bad\n", "0 0 3 7:nan\n", "0 0 3 7:1:2\n")
+ for text in invalid:
+ with self.subTest(text=text), tempfile.TemporaryDirectory() as tmp:
+ path = Path(tmp) / "trace.txt"
+ path.write_text(text, encoding="utf-8")
+ with self.assertRaisesRegex(ValueError, r"trace\.txt:1"):
+ sim.parse_trace(path)
+
+
+class SpecificationTest(unittest.TestCase):
+ def test_manifest_separates_resident_and_read_bytes(self):
+ traces = [[sim.AccessEvent(3, (0, 1))]]
+ with tempfile.TemporaryDirectory() as tmp:
+ path = Path(tmp) / "manifest.json"
+ path.write_text(json.dumps({"layers": {"3": {
+ "resident_bytes": 123,
+ "read_bytes": 456,
+ "felt_miss_us": 45.5,
+ "max_experts": 9,
+ }}}), encoding="utf-8")
+ specs = sim.load_specs(traces, path, 1.0, 1.0)
+ self.assertEqual(specs[3], sim.LayerSpec(123, 456, 45.5, 9))
+
+ def test_manifest_bounds_are_required_for_observed_layers_and_experts(self):
+ traces = [[sim.AccessEvent(3, (9,))]]
+ with tempfile.TemporaryDirectory() as tmp:
+ path = Path(tmp) / "manifest.json"
+ path.write_text(json.dumps({"layers": {"3": {
+ "resident_bytes": 123,
+ "max_experts": 9,
+ }}}), encoding="utf-8")
+ with self.assertRaisesRegex(ValueError, "exceeds manifest bound"):
+ sim.load_specs(traces, path, 1.0, 1.0)
+
+ def test_default_felt_cost_uses_read_bytes(self):
+ traces = [[sim.AccessEvent(0, (0,))]]
+ with tempfile.TemporaryDirectory() as tmp:
+ path = Path(tmp) / "manifest.json"
+ path.write_text(json.dumps({"layers": {"0": {
+ "resident_bytes": 10,
+ "read_bytes": 1_000_000_000,
+ "max_experts": 1,
+ }}}), encoding="utf-8")
+ specs = sim.load_specs(traces, path, 2.0, 0.5)
+ self.assertEqual(specs[0].felt_miss_us, 250_000)
+
+ def test_manifest_layers_absent_from_traces_do_not_consume_budget(self):
+ traces = [[sim.AccessEvent(0, (0,))]]
+ with tempfile.TemporaryDirectory() as tmp:
+ path = Path(tmp) / "manifest.json"
+ path.write_text(json.dumps({"layers": {
+ "0": {"resident_bytes": 10, "max_experts": 1},
+ "1": {"resident_bytes": 10, "max_experts": 1},
+ }}), encoding="utf-8")
+ specs = sim.load_specs(traces, path, 1.0, 1.0)
+ self.assertEqual(set(specs), {0})
+
+ def test_profile_costs_extracts_aggregate_felt_wait(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ path = Path(tmp) / "profile.out"
+ path.write_text(
+ "[PROF] expert I/O: 38.178 GB fetched | 525.0 loads/token | "
+ "7.2s read service / 1.4s felt wait\n",
+ encoding="utf-8",
+ )
+ costs = sim.profile_costs([path])
+ self.assertEqual(costs["loads"], 525)
+ self.assertEqual(costs["felt_wait_seconds"], 1.4)
+ self.assertAlmostEqual(costs["felt_us_per_load"], 2666.6666667)
+
+
+class PolicyBehaviorTest(unittest.TestCase):
+ def specs(self):
+ return {0: sim.LayerSpec(1, 4, 10, 100)}
+
+ def events(self, sequence):
+ return [sim.AccessEvent(0, (expert,), (expert,)) for expert in sequence]
+
+ def test_lru_known_sequence_and_read_bytes(self):
+ stats = sim.run_policy("lru", {0: 2}, self.specs(),
+ self.events([1, 2, 1, 3, 1, 2]))
+ self.assertEqual((stats.hits, stats.misses), (2, 4))
+ self.assertEqual(stats.bytes_read, 16)
+ self.assertEqual(stats.felt_wait_us, 40)
+
+ def test_frequency_admission_rejects_scan_pollution(self):
+ warm = [0, 1] * 20
+ sequence = warm[:]
+ for cold in range(2, 22):
+ sequence.extend([cold, 0, 1])
+ lru = sim.run_policy("lru", {0: 2}, self.specs(), self.events(sequence))
+ frequency = sim.run_policy(
+ "frequency", {0: 2}, self.specs(), self.events(sequence))
+ self.assertGreater(frequency.hits, lru.hits)
+ self.assertLess(frequency.misses, lru.misses)
+ self.assertGreater(frequency.rejected_admissions, 0)
+
+ def test_training_uses_row_selection_multiplicity(self):
+ event = sim.AccessEvent(0, (7, 8), (7, 7, 7, 8))
+ counts = sim.training_counts([[event]])
+ self.assertEqual(counts[0], {7: 3, 8: 1})
+
+ def test_training_counts_seed_frequency_and_half_pinned_policies(self):
+ learned = {0: {7: 100, 8: 50, 9: 1}}
+ first = self.events([7, 8])
+ frequency = sim.run_policy(
+ "frequency", {0: 2}, self.specs(), first, learned)
+ pinned = sim.run_policy(
+ "half-pinned", {0: 2}, self.specs(), first, learned)
+ self.assertEqual(frequency.hits, 2)
+ self.assertEqual(frequency.seeded_objects, 2)
+ self.assertEqual(pinned.hits, 1)
+ self.assertEqual(pinned.seeded_objects, 1)
+
+ def test_slru_protects_reused_objects_from_one_hit_scan(self):
+ warm = [0, 1] * 20
+ sequence = warm + sum(([cold, 0, 1] for cold in range(2, 22)), [])
+ lru = sim.run_policy("lru", {0: 3}, self.specs(), self.events(sequence))
+ slru = sim.run_policy("slru", {0: 3}, self.specs(), self.events(sequence))
+ self.assertGreaterEqual(slru.hits, lru.hits)
+
+ def test_lru_promotes_between_64_expert_blocks(self):
+ event = sim.AccessEvent(0, tuple(range(65)) + (0,))
+ stats = sim.run_policy("lru", {0: 64}, self.specs(), [event])
+ self.assertEqual(stats.hits, 1)
+ self.assertEqual(stats.misses, 65)
+
+
+class AllocationTest(unittest.TestCase):
+ def test_uniform_allocation_respects_resident_bytes(self):
+ specs = {
+ 0: sim.LayerSpec(10, 100, 1, 8),
+ 1: sim.LayerSpec(20, 100, 1, 8),
+ }
+ caps = sim.uniform_capacities(specs, 95)
+ self.assertEqual(caps, {0: 3, 1: 3})
+ self.assertEqual(sim.capacity_bytes(caps, specs), 90)
+
+ def test_dynamic_budget_uses_exact_policy_replay(self):
+ specs = {
+ 0: sim.LayerSpec(1, 1, 1, 8),
+ 1: sim.LayerSpec(1, 1, 10, 8),
+ }
+ trace = [[
+ *[sim.AccessEvent(0, (expert,)) for expert in [0, 1, 2, 3] * 20],
+ *[sim.AccessEvent(1, (expert,)) for expert in [0, 1] * 40],
+ ]]
+ counts = sim.training_counts(trace)
+ caps = sim.dynamic_capacities(trace, specs, 4, "lru", counts)
+ self.assertGreater(caps[1], caps[0])
+ self.assertLessEqual(sim.capacity_bytes(caps, specs), 4)
+
+ def test_training_traces_are_independent_cold_runs(self):
+ specs = {0: sim.LayerSpec(1, 1, 1, 4)}
+ traces = [
+ [sim.AccessEvent(0, (1,))],
+ [sim.AccessEvent(0, (1,))],
+ ]
+ wait = sim._training_wait(traces, {0: 0}, specs, "lru", {})
+ self.assertEqual(wait, 2)
+
+ def test_shifted_heldout_dynamic_lru_regression_is_visible(self):
+ train, evaluation, specs, budget = sim.synthetic_trace("shifted")
+ result = sim.analyze(train, evaluation, specs, budget, ["lru"])
+ uniform = result["results"]["uniform"]["lru"]["aggregate"]
+ dynamic = result["results"]["dynamic-lru"]["lru"]["aggregate"]
+ self.assertGreater(dynamic["felt_wait_us"], uniform["felt_wait_us"])
+
+
+class DecisionGateTest(unittest.TestCase):
+ def result(self, category_waits):
+ def payload(values):
+ total = sum(values)
+ return {
+ "aggregate": {"felt_wait_us": total},
+ "traces": [{"felt_wait_us": value} for value in values],
+ }
+
+ return {"results": {
+ "uniform": {
+ "lru": payload([100, 100, 100]),
+ "frequency": payload(category_waits),
+ },
+ "dynamic-frequency": {
+ "frequency": payload([70, 70, 70]),
+ },
+ }}
+
+ def test_gate_equal_weights_categories_and_rejects_worst_regression(self):
+ categories = ["chat", "chat", "code"]
+ gate = sim.decision_gate(self.result([80, 80, 104]), categories)
+ frequency = next(
+ row for row in gate["candidates"]
+ if row["allocation"] == "uniform" and row["policy"] == "frequency")
+ self.assertAlmostEqual(frequency["category_mean_gain"], 0.08)
+ self.assertAlmostEqual(frequency["worst_category_gain"], -0.04)
+ self.assertFalse(frequency["pass"])
+ dynamic = next(
+ row for row in gate["candidates"]
+ if row["allocation"] == "dynamic-frequency")
+ self.assertTrue(dynamic["pass"])
+
+ def test_trace_categories_strip_numeric_replicates_or_use_explicit_names(self):
+ traces = [
+ [sim.AccessEvent(0, (0,), source="/tmp/chat_1.trace")],
+ [sim.AccessEvent(0, (0,), source="/tmp/chat_2.trace")],
+ [sim.AccessEvent(0, (0,), source="/tmp/code-1.trace")],
+ ]
+ self.assertEqual(sim.trace_categories(traces), ["chat", "chat", "code"])
+ self.assertEqual(
+ sim.trace_categories(traces, ["a", "b", "c"]), ["a", "b", "c"])
+ with self.assertRaisesRegex(ValueError, "one value per"):
+ sim.trace_categories(traces, ["a"])
+
+ def test_layer_specific_sensitivity_can_invalidate_candidate(self):
+ train, evaluation, specs, budget = sim.synthetic_trace("stationary")
+ sensitivity = sim.sensitivity_analysis(
+ train, evaluation, specs, budget, ["lru", "frequency"],
+ ["stationary"], [0.5, 1.0, 1.5])
+ frequency = next(
+ row for row in sensitivity["candidates"]
+ if row["allocation"] == "uniform" and row["policy"] == "frequency")
+ self.assertEqual(len(frequency["outcomes"]), 1 + 2 * len(specs))
+ self.assertTrue(any(
+ outcome["scenario"] == "layer-1-x0.5"
+ for outcome in frequency["outcomes"]))
+
+ def test_replay_cache_preserves_full_replay_metrics(self):
+ train, evaluation, specs, budget = sim.synthetic_trace("stationary")
+ counts = sim.training_counts(train)
+ capacities = sim.uniform_capacities(specs, budget)
+ cached = sim.evaluate(
+ evaluation, capacities, specs, ["lru", "frequency"], counts)
+ for policy in ("lru", "frequency"):
+ direct = sim.PolicyStats()
+ for trace in evaluation:
+ direct.add(sim.run_policy(policy, capacities, specs, trace, counts))
+ self.assertEqual(cached[policy]["aggregate"], direct.report())
+
+ def test_combined_category_suite_applies_worst_category_guard(self):
+ train, evaluation, specs, budget, categories = sim.synthetic_category_suite()
+ result = sim.analyze(train, evaluation, specs, budget, ["lru", "frequency"])
+ gate = sim.decision_gate(result, categories)
+ frequency = next(
+ candidate for candidate in gate["candidates"]
+ if candidate["allocation"] == "uniform"
+ and candidate["policy"] == "frequency")
+ self.assertGreater(frequency["category_mean_gain"], 0.10)
+ self.assertLess(frequency["worst_category_gain"], -0.03)
+
+ def test_synthetic_budget_sweep_exposes_robust_and_fragile_regions(self):
+ rows = sim.synthetic_budget_sweep(["lru", "frequency"], [2, 8])
+ frequency = {
+ row["budget_mb"]: row
+ for row in rows
+ if row["allocation"] == "dynamic-frequency"
+ }
+ self.assertTrue(frequency[2]["pass_sensitivity"])
+ self.assertFalse(frequency[8]["pass_sensitivity"])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/c/tools/residency_sim.py b/c/tools/residency_sim.py
new file mode 100644
index 00000000..a4e8a822
--- /dev/null
+++ b/c/tools/residency_sim.py
@@ -0,0 +1,1177 @@
+#!/usr/bin/env python3
+"""Offline GLM expert-residency policy simulator.
+
+The simulator consumes Colibri ROUTE_TRACE output without loading a model. It
+replays GLM's routed-expert unions through bounded per-layer caches and compares
+placement policies under the same expert-residency byte budget. It deliberately
+models demand residency only: no speculative reads, routing changes, or token
+semantics are involved.
+
+ROUTE_TRACE lines have this form (one row per token position and MoE call):
+
+ <call> <row> <layer> <expert>:<gate> ...
+
+Rows sharing one contiguous (call, layer) group are a runtime batch. The parser
+retains both the first-seen expert union for demand replay and per-row selection
+counts for training startup residents.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import re
+import statistics
+from collections import Counter, OrderedDict, defaultdict
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Sequence
+
+
+EXPERT_FIELD = re.compile(r"^(\d+):([^:]+)$")
+
+
+@dataclass(frozen=True)
+class AccessEvent:
+ layer: int
+ experts: tuple[int, ...]
+ selections: tuple[int, ...] = ()
+ call: int | None = None
+ source: str = ""
+
+ def selection_counts(self) -> Counter[int]:
+ return Counter(self.selections or self.experts)
+
+
+@dataclass(frozen=True)
+class LayerSpec:
+ resident_bytes: int
+ read_bytes: int
+ felt_miss_us: float
+ max_experts: int
+
+
+@dataclass
+class PolicyStats:
+ requests: int = 0
+ hits: int = 0
+ misses: int = 0
+ bytes_read: int = 0
+ felt_wait_us: float = 0.0
+ admissions: int = 0
+ rejected_admissions: int = 0
+ evictions: int = 0
+ seeded_objects: int = 0
+
+ def add(self, other: "PolicyStats") -> None:
+ for field in self.__dataclass_fields__:
+ setattr(self, field, getattr(self, field) + getattr(other, field))
+
+ def report(self) -> dict:
+ accesses = self.hits + self.misses
+ return {
+ **asdict(self),
+ "accesses": accesses,
+ "hit_rate": self.hits / accesses if accesses else 1.0,
+ }
+
+
+def _build_event(key, union, selections, source):
+ if key is None or not union:
+ return None
+ return AccessEvent(key[1], tuple(union), tuple(selections), key[0], source)
+
+
+def parse_trace(path: Path) -> list[AccessEvent]:
+ """Parse contiguous GLM ROUTE_TRACE call groups into demand-union events."""
+ events = []
+ key = None
+ union: list[int] = []
+ known: set[int] = set()
+ selections: list[int] = []
+ seen_keys: set[tuple[int, int]] = set()
+ next_row = 0
+ last_call = -1
+ for lineno, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
+ fields = raw.split()
+ if not fields:
+ continue
+ if len(fields) < 4:
+ raise ValueError(f"{path}:{lineno}: expected call row layer expert:gate ...")
+ try:
+ call, row, layer = map(int, fields[:3])
+ except ValueError as error:
+ raise ValueError(f"{path}:{lineno}: invalid call/row/layer") from error
+ if call < 0 or row < 0 or layer < 0:
+ raise ValueError(f"{path}:{lineno}: negative call/row/layer")
+ next_key = (call, layer)
+ if next_key != key:
+ event = _build_event(key, union, selections, str(path))
+ if event:
+ events.append(event)
+ if next_key in seen_keys:
+ raise ValueError(f"{path}:{lineno}: noncontiguous call/layer group {next_key}")
+ if call < last_call:
+ raise ValueError(f"{path}:{lineno}: call id moved backwards")
+ seen_keys.add(next_key)
+ key = next_key
+ union = []
+ known = set()
+ selections = []
+ next_row = 0
+ last_call = call
+ if row != next_row:
+ raise ValueError(
+ f"{path}:{lineno}: expected row {next_row} in call/layer group, got {row}")
+ next_row += 1
+ for value in fields[3:]:
+ match = EXPERT_FIELD.fullmatch(value)
+ if not match:
+ raise ValueError(f"{path}:{lineno}: invalid expert field {value!r}")
+ expert = int(match.group(1))
+ try:
+ gate = float(match.group(2))
+ except ValueError as error:
+ raise ValueError(f"{path}:{lineno}: invalid gate in {value!r}") from error
+ if not math.isfinite(gate):
+ raise ValueError(f"{path}:{lineno}: non-finite gate in {value!r}")
+ selections.append(expert)
+ if expert not in known:
+ known.add(expert)
+ union.append(expert)
+ event = _build_event(key, union, selections, str(path))
+ if event:
+ events.append(event)
+ return events
+
+
+def validate_glm_trace(events: Sequence[AccessEvent], path: Path) -> None:
+ if not events or any(event.call is None for event in events):
+ return
+ calls = [event.call for event in events]
+ if calls[0] != 0 or any(
+ call != previous + 1 for previous, call in zip(calls, calls[1:])):
+ raise ValueError(
+ f"{path}: trace lacks advancing GLM call ids; unsupported engine or corrupt trace")
+
+
+def read_traces(paths: Sequence[Path]) -> list[list[AccessEvent]]:
+ traces = [parse_trace(path) for path in paths]
+ if not traces or any(not trace for trace in traces):
+ raise ValueError("every trace must contain at least one routing event")
+ for path, trace in zip(paths, traces):
+ validate_glm_trace(trace, path)
+ return traces
+
+
+def load_specs(
+ traces: Sequence[Sequence[AccessEvent]],
+ manifest: Path,
+ read_gbps: float,
+ felt_fraction: float,
+) -> dict[int, LayerSpec]:
+ if not math.isfinite(read_gbps) or read_gbps <= 0:
+ raise ValueError("read GB/s must be positive and finite")
+ if not math.isfinite(felt_fraction) or not 0 <= felt_fraction <= 1:
+ raise ValueError("felt fraction must be finite and in [0,1]")
+ document = json.loads(manifest.read_text(encoding="utf-8"))
+ if not isinstance(document, dict) or not isinstance(document.get("layers"), dict):
+ raise ValueError("manifest.layers must be an object")
+ specs = {}
+ for raw_layer, raw_spec in document["layers"].items():
+ try:
+ layer = int(raw_layer)
+ except (TypeError, ValueError) as error:
+ raise ValueError(f"invalid manifest layer {raw_layer!r}") from error
+ if layer < 0 or not isinstance(raw_spec, dict):
+ raise ValueError(f"invalid layer {raw_layer!r} specification")
+ try:
+ resident_bytes = int(raw_spec["resident_bytes"])
+ read_bytes = int(raw_spec.get("read_bytes", resident_bytes))
+ max_experts = int(raw_spec["max_experts"])
+ default_felt = (read_bytes / (read_gbps * 1e9) * 1e6 * felt_fraction)
+ felt = float(raw_spec.get("felt_miss_us", default_felt))
+ except (KeyError, TypeError, ValueError) as error:
+ raise ValueError(f"invalid layer {layer} specification") from error
+ if (resident_bytes <= 0 or read_bytes <= 0 or max_experts <= 0 or
+ felt < 0 or not math.isfinite(felt)):
+ raise ValueError(f"invalid layer {layer} specification")
+ specs[layer] = LayerSpec(resident_bytes, read_bytes, felt, max_experts)
+ observed = defaultdict(set)
+ for trace in traces:
+ for event in trace:
+ observed[event.layer].update(event.experts)
+ for layer, experts in observed.items():
+ if layer not in specs:
+ raise ValueError(f"trace layer {layer} is absent from manifest")
+ invalid = [expert for expert in experts if expert >= specs[layer].max_experts]
+ if invalid:
+ raise ValueError(f"trace layer {layer} expert {min(invalid)} exceeds manifest bound")
+ return {layer: spec for layer, spec in specs.items() if layer in observed}
+
+
+def uniform_capacities(specs: dict[int, LayerSpec], budget_bytes: int) -> dict[int, int]:
+ """Allocate equal slot counts per layer, bounded by each layer's expert count."""
+ if budget_bytes < 0:
+ raise ValueError("budget must be non-negative")
+ capacities = {layer: 0 for layer in specs}
+ used = 0
+ while True:
+ cost = sum(spec.resident_bytes for layer, spec in specs.items()
+ if capacities[layer] < spec.max_experts)
+ if cost <= 0 or used + cost > budget_bytes:
+ break
+ for layer, spec in specs.items():
+ if capacities[layer] < spec.max_experts:
+ capacities[layer] += 1
+ used += spec.resident_bytes
+ return capacities
+
+
+class BasePolicy:
+ def __init__(self, capacities, specs, learned_counts=None):
+ self.capacities = capacities
+ self.specs = specs
+ self.learned_counts = learned_counts or {}
+ self.stats = PolicyStats()
+
+ def lookup(self, layer: int, expert: int) -> bool:
+ raise NotImplementedError
+
+ def observe(self, layer: int, expert: int, hit: bool) -> None:
+ pass
+
+ def admit_batch(self, layer: int, misses: Sequence[int]) -> None:
+ raise NotImplementedError
+
+ def access(self, event: AccessEvent) -> None:
+ # GLM resolves/promotes at most 64 experts at a time.
+ for start in range(0, len(event.experts), 64):
+ misses = []
+ spec = self.specs[event.layer]
+ for expert in event.experts[start:start + 64]:
+ self.stats.requests += 1
+ hit = self.lookup(event.layer, expert)
+ self.observe(event.layer, expert, hit)
+ if hit:
+ self.stats.hits += 1
+ else:
+ self.stats.misses += 1
+ self.stats.bytes_read += spec.read_bytes
+ self.stats.felt_wait_us += spec.felt_miss_us
+ misses.append(expert)
+ self.admit_batch(event.layer, misses)
+
+
+class LRUPolicy(BasePolicy):
+ def __init__(self, capacities, specs, learned_counts=None):
+ super().__init__(capacities, specs, learned_counts)
+ self.cache: dict[int, OrderedDict[int, None]] = defaultdict(OrderedDict)
+
+ def lookup(self, layer, expert):
+ cache = self.cache[layer]
+ if expert not in cache:
+ return False
+ cache.move_to_end(expert)
+ return True
+
+ def admit_batch(self, layer, misses):
+ capacity = self.capacities.get(layer, 0)
+ if capacity <= 0:
+ self.stats.rejected_admissions += len(misses)
+ return
+ selected = list(misses[-capacity:])
+ self.stats.rejected_admissions += len(misses) - len(selected)
+ cache = self.cache[layer]
+ for expert in reversed(selected):
+ if expert in cache:
+ cache.move_to_end(expert)
+ continue
+ while len(cache) >= capacity:
+ cache.popitem(last=False)
+ self.stats.evictions += 1
+ cache[expert] = None
+ self.stats.admissions += 1
+
+
+class LFUPolicy(BasePolicy):
+ """Always-admit LFU with per-layer periodic decay and recency ties."""
+ def __init__(self, capacities, specs, learned_counts=None,
+ decay_interval=4096, decay=0.5):
+ super().__init__(capacities, specs, learned_counts)
+ self.resident: dict[int, set[int]] = defaultdict(set)
+ self.frequency: dict[int, Counter[int]] = defaultdict(Counter)
+ self.last: dict[int, dict[int, int]] = defaultdict(dict)
+ self.clock: Counter[int] = Counter()
+ self.decay_interval = decay_interval
+ self.decay = decay
+ for layer, capacity in capacities.items():
+ counts = Counter(self.learned_counts.get(layer, {}))
+ self.frequency[layer].update(counts)
+ ranked = sorted(counts, key=lambda expert: (-counts[expert], expert))[:capacity]
+ self.resident[layer].update(ranked)
+ self.stats.seeded_objects += len(ranked)
+
+ def lookup(self, layer, expert):
+ return expert in self.resident[layer]
+
+ def observe(self, layer, expert, hit):
+ self.clock[layer] += 1
+ if self.decay_interval and self.clock[layer] % self.decay_interval == 0:
+ for candidate in list(self.frequency[layer]):
+ self.frequency[layer][candidate] *= self.decay
+ if self.frequency[layer][candidate] < 0.5:
+ del self.frequency[layer][candidate]
+ self.frequency[layer][expert] += 1
+ self.last[layer][expert] = self.clock[layer]
+
+ def _victim(self, layer):
+ return min(self.resident[layer],
+ key=lambda expert: (self.frequency[layer][expert],
+ self.last[layer].get(expert, 0), expert))
+
+ def admit_batch(self, layer, misses):
+ capacity = self.capacities.get(layer, 0)
+ if capacity <= 0:
+ self.stats.rejected_admissions += len(misses)
+ return
+ resident = self.resident[layer]
+ for expert in misses:
+ if expert in resident:
+ continue
+ if len(resident) >= capacity:
+ resident.remove(self._victim(layer))
+ self.stats.evictions += 1
+ resident.add(expert)
+ self.stats.admissions += 1
+
+
+class SLRUPolicy(BasePolicy):
+ """Segmented LRU: one-hit objects enter probation; reused objects are protected."""
+ def __init__(self, capacities, specs, learned_counts=None, protected_fraction=0.5):
+ super().__init__(capacities, specs, learned_counts)
+ self.protected_fraction = protected_fraction
+ self.probation: dict[int, OrderedDict[int, None]] = defaultdict(OrderedDict)
+ self.protected: dict[int, OrderedDict[int, None]] = defaultdict(OrderedDict)
+
+ def _limits(self, layer):
+ capacity = self.capacities.get(layer, 0)
+ if capacity <= 1:
+ return capacity, 0
+ protected = min(capacity - 1, max(1, int(capacity * self.protected_fraction)))
+ return capacity - protected, protected
+
+ def lookup(self, layer, expert):
+ protected = self.protected[layer]
+ probation = self.probation[layer]
+ if expert in protected:
+ protected.move_to_end(expert)
+ return True
+ if expert not in probation:
+ return False
+ del probation[expert]
+ probation_limit, protected_limit = self._limits(layer)
+ if protected_limit:
+ while len(protected) >= protected_limit:
+ demoted, _ = protected.popitem(last=False)
+ probation[demoted] = None
+ while len(probation) > probation_limit:
+ probation.popitem(last=False)
+ self.stats.evictions += 1
+ protected[expert] = None
+ else:
+ probation[expert] = None
+ return True
+
+ def admit_batch(self, layer, misses):
+ capacity = self.capacities.get(layer, 0)
+ probation_limit, _protected_limit = self._limits(layer)
+ if capacity <= 0:
+ self.stats.rejected_admissions += len(misses)
+ return
+ probation = self.probation[layer]
+ protected = self.protected[layer]
+ for expert in misses:
+ if expert in probation or expert in protected:
+ continue
+ probation[expert] = None
+ self.stats.admissions += 1
+ while len(probation) > probation_limit:
+ probation.popitem(last=False)
+ self.stats.evictions += 1
+
+
+class FrequencyAdmissionPolicy(BasePolicy):
+ """Decayed-frequency admission: demand may execute without polluting cache."""
+ def __init__(self, capacities, specs, learned_counts=None,
+ decay_interval=4096, decay=0.5, hysteresis=0.10):
+ super().__init__(capacities, specs, learned_counts)
+ self.resident: dict[int, set[int]] = defaultdict(set)
+ self.frequency: dict[int, Counter[int]] = defaultdict(Counter)
+ self.last: dict[int, dict[int, int]] = defaultdict(dict)
+ self.clock: Counter[int] = Counter()
+ self.decay_interval = decay_interval
+ self.decay = decay
+ self.hysteresis = hysteresis
+ for layer, capacity in capacities.items():
+ counts = Counter(self.learned_counts.get(layer, {}))
+ self.frequency[layer].update(counts)
+ ranked = sorted(counts, key=lambda expert: (-counts[expert], expert))[:capacity]
+ self.resident[layer].update(ranked)
+ self.stats.seeded_objects += len(ranked)
+
+ def lookup(self, layer, expert):
+ return expert in self.resident[layer]
+
+ def observe(self, layer, expert, hit):
+ self.clock[layer] += 1
+ if self.decay_interval and self.clock[layer] % self.decay_interval == 0:
+ for candidate in list(self.frequency[layer]):
+ self.frequency[layer][candidate] *= self.decay
+ if self.frequency[layer][candidate] < 0.5:
+ del self.frequency[layer][candidate]
+ self.frequency[layer][expert] += 1
+ self.last[layer][expert] = self.clock[layer]
+
+ def _victim(self, layer):
+ return min(self.resident[layer],
+ key=lambda expert: (self.frequency[layer][expert],
+ self.last[layer].get(expert, 0), expert))
+
+ def admit_batch(self, layer, misses):
+ capacity = self.capacities.get(layer, 0)
+ if capacity <= 0:
+ self.stats.rejected_admissions += len(misses)
+ return
+ resident = self.resident[layer]
+ for expert in misses:
+ if expert in resident:
+ continue
+ if len(resident) < capacity:
+ resident.add(expert)
+ self.stats.admissions += 1
+ continue
+ victim = self._victim(layer)
+ if (self.frequency[layer][expert] <=
+ self.frequency[layer][victim] * (1 + self.hysteresis)):
+ self.stats.rejected_admissions += 1
+ continue
+ resident.remove(victim)
+ resident.add(expert)
+ self.stats.evictions += 1
+ self.stats.admissions += 1
+
+
+class HalfPinnedLRUPolicy(BasePolicy):
+ """Synthetic half-capacity frequency pins plus adaptive LRU."""
+ def __init__(self, capacities, specs, learned_counts=None, pin_fraction=0.5):
+ super().__init__(capacities, specs, learned_counts)
+ self.pins: dict[int, set[int]] = defaultdict(set)
+ self.cache: dict[int, OrderedDict[int, None]] = defaultdict(OrderedDict)
+ self.adaptive_capacity = {}
+ for layer, capacity in capacities.items():
+ pin_count = min(capacity, int(capacity * pin_fraction))
+ counts = self.learned_counts.get(layer, {})
+ ranked = sorted(counts, key=lambda expert: (-counts[expert], expert))[:pin_count]
+ self.pins[layer].update(ranked)
+ self.adaptive_capacity[layer] = capacity - len(ranked)
+ self.stats.seeded_objects += len(ranked)
+
+ def lookup(self, layer, expert):
+ if expert in self.pins[layer]:
+ return True
+ cache = self.cache[layer]
+ if expert not in cache:
+ return False
+ cache.move_to_end(expert)
+ return True
+
+ def admit_batch(self, layer, misses):
+ capacity = self.adaptive_capacity.get(layer, 0)
+ if capacity <= 0:
+ self.stats.rejected_admissions += len(misses)
+ return
+ selected = list(misses[-capacity:])
+ self.stats.rejected_admissions += len(misses) - len(selected)
+ cache = self.cache[layer]
+ for expert in reversed(selected):
+ if expert in self.pins[layer] or expert in cache:
+ continue
+ while len(cache) >= capacity:
+ cache.popitem(last=False)
+ self.stats.evictions += 1
+ cache[expert] = None
+ self.stats.admissions += 1
+
+
+POLICIES = {
+ "lru": LRUPolicy,
+ "half-pinned": HalfPinnedLRUPolicy,
+ "lfu": LFUPolicy,
+ "slru": SLRUPolicy,
+ "frequency": FrequencyAdmissionPolicy,
+}
+
+
+def training_counts(traces):
+ counts: dict[int, Counter[int]] = defaultdict(Counter)
+ for trace in traces:
+ for event in trace:
+ counts[event.layer].update(event.selection_counts())
+ return dict(counts)
+
+
+def run_policy(policy_name, capacities, specs, trace, learned_counts=None):
+ policy = POLICIES[policy_name](capacities, specs, learned_counts)
+ for event in trace:
+ policy.access(event)
+ return policy.stats
+
+
+class LayerReplayCache:
+ """Memoize exact per-layer replays; all implemented policies are layer-local."""
+ def __init__(self, traces, specs, learned_counts=None, base=None):
+ self.traces = traces
+ self.specs = specs
+ self.learned_counts = learned_counts or {}
+ self.layer_traces = {
+ layer: [[event for event in trace if event.layer == layer] for trace in traces]
+ for layer in specs
+ }
+ self.cache = base.cache if base is not None else {}
+
+ def layer_stats(self, policy_name, layer, capacity, trace_index):
+ key = (policy_name, layer, capacity, trace_index)
+ if key not in self.cache:
+ self.cache[key] = run_policy(
+ policy_name,
+ {layer: capacity},
+ {layer: self.specs[layer]},
+ self.layer_traces[layer][trace_index],
+ {layer: self.learned_counts.get(layer, {})},
+ )
+ return self.cache[key]
+
+ def trace_stats(self, policy_name, capacities, specs, trace_index):
+ combined = PolicyStats()
+ felt_wait_us = 0.0
+ for layer, spec in specs.items():
+ stats = self.layer_stats(
+ policy_name, layer, capacities.get(layer, 0), trace_index)
+ combined.add(stats)
+ felt_wait_us += stats.misses * spec.felt_miss_us
+ combined.felt_wait_us = felt_wait_us
+ return combined
+
+ def layer_wait(self, policy_name, layer, capacity, specs):
+ misses = sum(
+ self.layer_stats(policy_name, layer, capacity, trace_index).misses
+ for trace_index in range(len(self.traces))
+ )
+ return misses * specs[layer].felt_miss_us
+
+
+class ScaledReplayCache(LayerReplayCache):
+ def __init__(self, base, specs):
+ super().__init__(base.traces, specs, base.learned_counts, base=base)
+
+
+def evaluate(
+ traces,
+ capacities,
+ specs,
+ policies,
+ learned_counts=None,
+ replay_cache=None,
+) -> dict:
+ replay_cache = replay_cache or LayerReplayCache(traces, specs, learned_counts)
+ result = {}
+ for policy_name in policies:
+ aggregate = PolicyStats()
+ per_trace = []
+ for trace_index, trace in enumerate(traces):
+ stats = replay_cache.trace_stats(
+ policy_name, capacities, specs, trace_index)
+ aggregate.add(stats)
+ per_trace.append({
+ "source": trace[0].source if trace else "",
+ **stats.report(),
+ })
+ result[policy_name] = {"aggregate": aggregate.report(), "traces": per_trace}
+ return result
+
+
+def capacity_bytes(capacities, specs):
+ return sum(capacities.get(layer, 0) * spec.resident_bytes
+ for layer, spec in specs.items())
+
+
+def scale_specs(specs, felt_scale, target_layer=None):
+ if felt_scale <= 0 or not math.isfinite(felt_scale):
+ raise ValueError("felt scale must be positive and finite")
+ return {
+ layer: LayerSpec(
+ spec.resident_bytes,
+ spec.read_bytes,
+ (spec.felt_miss_us * felt_scale
+ if target_layer is None or layer == target_layer else spec.felt_miss_us),
+ spec.max_experts,
+ )
+ for layer, spec in specs.items()
+ }
+
+
+def trace_categories(traces, category_names=None):
+ if category_names is not None:
+ if len(category_names) != len(traces):
+ raise ValueError("--eval-category requires one value per --eval trace")
+ return list(category_names)
+ categories = []
+ for trace in traces:
+ source = Path(trace[0].source).stem if trace and trace[0].source else "trace"
+ match = re.match(r"^(.*?)(?:[_-]\d+)?$", source)
+ categories.append(match.group(1) or source)
+ return categories
+
+
+def profile_costs(log_paths):
+ """Extract per-layer felt costs from PROF logs when layer telemetry exists.
+
+ Current GLM PROF output reports aggregate felt wait, so return the aggregate
+ cost-per-miss fallback and leave per-layer calibration to a future layer
+ telemetry field.
+ """
+ waits = []
+ loads = []
+ for path in log_paths or ():
+ text = Path(path).read_text(encoding="utf-8")
+ for line in text.splitlines():
+ match = re.search(r"([0-9.]+)s read service / ([0-9.]+)s felt wait", line)
+ load_match = re.search(r"\| ([0-9]+(?:\.[0-9]+)?) loads/token", line)
+ if match:
+ waits.append(float(match.group(2)))
+ if load_match:
+ loads.append(float(load_match.group(1)))
+ return {
+ "logs": [str(path) for path in log_paths or ()],
+ "felt_wait_seconds": sum(waits),
+ "loads": sum(loads),
+ "felt_us_per_load": (sum(waits) * 1e6 / sum(loads) if loads else None),
+ "scope": "aggregate; GLM PROF does not expose layer-level felt wait",
+ }
+
+
+def decision_gate(result, categories, minimum_gain=0.10, maximum_regression=0.03):
+ """Compare every candidate with uniform LRU using equal-weight categories."""
+ if not 0 <= minimum_gain < 1 or maximum_regression < 0:
+ raise ValueError("invalid decision-gate thresholds")
+ baseline_payload = result["results"].get("uniform", {}).get("lru")
+ if not baseline_payload:
+ raise ValueError("decision gate requires policy lru")
+ if len(baseline_payload["traces"]) != len(categories):
+ raise ValueError("category count does not match evaluation traces")
+
+ baseline_by_category: dict[str, float] = defaultdict(float)
+ category_trace_counts: Counter[str] = Counter(categories)
+ for category, stats in zip(categories, baseline_payload["traces"]):
+ baseline_by_category[category] += stats["felt_wait_us"]
+ rows = []
+ for allocation, policies in result["results"].items():
+ for policy, payload in policies.items():
+ if allocation == "uniform" and policy == "lru":
+ continue
+ candidate_by_category: dict[str, float] = defaultdict(float)
+ for category, stats in zip(categories, payload["traces"]):
+ candidate_by_category[category] += stats["felt_wait_us"]
+ category_changes = {}
+ for category in sorted(baseline_by_category):
+ baseline = baseline_by_category[category]
+ candidate = candidate_by_category[category]
+ change = ((baseline - candidate) / baseline
+ if baseline else (0.0 if candidate == 0 else -math.inf))
+ category_changes[category] = {
+ "baseline_felt_wait_us": baseline,
+ "candidate_felt_wait_us": candidate,
+ "gain": change,
+ "traces": category_trace_counts[category],
+ }
+ gains = [entry["gain"] for entry in category_changes.values()]
+ mean_gain = statistics.fmean(gains)
+ worst_gain = min(gains)
+ aggregate_baseline = baseline_payload["aggregate"]["felt_wait_us"]
+ aggregate_candidate = payload["aggregate"]["felt_wait_us"]
+ aggregate_gain = ((aggregate_baseline - aggregate_candidate) / aggregate_baseline
+ if aggregate_baseline else 0.0)
+ row = {
+ "allocation": allocation,
+ "policy": policy,
+ "aggregate_gain": aggregate_gain,
+ "category_mean_gain": mean_gain,
+ "worst_category_gain": worst_gain,
+ "categories": category_changes,
+ }
+ row["pass"] = mean_gain >= minimum_gain and worst_gain >= -maximum_regression
+ rows.append(row)
+ rows.sort(key=lambda row: (
+ not row["pass"], -row["worst_category_gain"],
+ -row["category_mean_gain"], row["allocation"], row["policy"]))
+ return {
+ "baseline": {"allocation": "uniform", "policy": "lru"},
+ "minimum_category_mean_gain": minimum_gain,
+ "maximum_category_regression": maximum_regression,
+ "candidates": rows,
+ }
+
+
+def sensitivity_analysis(
+ train,
+ evaluation,
+ specs,
+ budget_bytes,
+ policies,
+ categories,
+ felt_scales,
+ minimum_gain=0.10,
+ maximum_regression=0.03,
+ sensitivity_layers=None,
+):
+ learned_counts = training_counts(train)
+ # Admission and hit/miss behavior is unchanged by a felt-cost scale within
+ # one layer. Reuse the exact replay curves across perturbations and only
+ # reweight their misses when optimizing dynamic capacity.
+ train_cache = LayerReplayCache(train, specs, learned_counts)
+ evaluation_cache = LayerReplayCache(evaluation, specs, learned_counts)
+ perturbations = [("baseline", None, 1.0)]
+ perturbations.extend(
+ (f"layer-{layer}-x{felt_scale:g}", layer, felt_scale)
+ for layer in (sorted(specs) if sensitivity_layers is None
+ else sorted(set(sensitivity_layers)))
+ if layer in specs
+ for felt_scale in felt_scales
+ if felt_scale != 1.0
+ )
+ scenarios = []
+ for name, layer, felt_scale in perturbations:
+ scenario_specs = scale_specs(specs, felt_scale, layer)
+ scenario_train_cache = ScaledReplayCache(train_cache, scenario_specs)
+ scenario_evaluation_cache = ScaledReplayCache(evaluation_cache, scenario_specs)
+ result = analyze(
+ train, evaluation, scenario_specs, budget_bytes, policies,
+ learned_counts, scenario_train_cache, scenario_evaluation_cache)
+ scenarios.append({
+ "name": name,
+ "layer": layer,
+ "felt_scale": felt_scale,
+ "gate": decision_gate(
+ result, categories, minimum_gain, maximum_regression),
+ })
+ candidates: dict[tuple[str, str], list[dict]] = defaultdict(list)
+ for scenario in scenarios:
+ for candidate in scenario["gate"]["candidates"]:
+ candidates[(candidate["allocation"], candidate["policy"])].append({
+ "scenario": scenario["name"],
+ "felt_scale": scenario["felt_scale"],
+ "pass": candidate["pass"],
+ "category_mean_gain": candidate["category_mean_gain"],
+ "worst_category_gain": candidate["worst_category_gain"],
+ })
+ robust = []
+ for (allocation, policy), outcomes in candidates.items():
+ robust.append({
+ "allocation": allocation,
+ "policy": policy,
+ "pass_all_scales": all(outcome["pass"] for outcome in outcomes),
+ "outcomes": outcomes,
+ })
+ robust.sort(key=lambda row: (not row["pass_all_scales"], row["allocation"], row["policy"]))
+ return {"scenarios": scenarios, "candidates": robust}
+
+
+def _training_wait(traces, capacities, specs, policy_name, learned_counts):
+ return sum(run_policy(policy_name, capacities, specs, trace, learned_counts).felt_wait_us
+ for trace in traces)
+
+
+def dynamic_capacities(
+ traces,
+ specs,
+ budget_bytes,
+ policy_name,
+ learned_counts,
+ replay_cache=None,
+):
+ """Greedily allocate slots using exact cold-trace replay for one policy."""
+ if budget_bytes < 0:
+ raise ValueError("budget must be non-negative")
+ capacities = {layer: 0 for layer in specs}
+ used = 0
+ replay_cache = replay_cache or LayerReplayCache(traces, specs, learned_counts)
+ layer_waits = {
+ layer: replay_cache.layer_wait(policy_name, layer, 0, specs)
+ for layer in specs
+ }
+ while True:
+ best = None
+ for layer, spec in specs.items():
+ current = capacities[layer]
+ # Exact replay is retained for every candidate, but scanning every
+ # slot is prohibitively expensive on a 256-expert x 75-layer model.
+ # These fixed breakpoints expose the useful capacity regions while
+ # keeping the offline sweep bounded and deterministic.
+ targets = {1, 2, 4, 8, 16, 32, 48, 64, 96, 128, 160, 192, 256}
+ targets.add(spec.max_experts)
+ for trace_index in range(len(traces)):
+ sequence = replay_cache.layer_traces[layer][trace_index]
+ unique = len({expert for event in sequence for expert in event.experts})
+ targets.add(min(spec.max_experts, unique))
+ targets = {target for target in targets
+ if current < target <= spec.max_experts}
+ for target in sorted(targets):
+ if target <= current:
+ continue
+ added_bytes = (target - current) * spec.resident_bytes
+ if used + added_bytes > budget_bytes:
+ break
+ wait = replay_cache.layer_wait(policy_name, layer, target, specs)
+ benefit = layer_waits[layer] - wait
+ candidate = (benefit / added_bytes, benefit, -added_bytes,
+ -layer, layer, target, wait, added_bytes)
+ if best is None or candidate > best:
+ best = candidate
+ if best is None or best[1] <= 0:
+ break
+ (_ratio, _benefit, _neg_bytes, _neg_layer, layer, target,
+ layer_wait, added_bytes) = best
+ capacities[layer] = target
+ layer_waits[layer] = layer_wait
+ used += added_bytes
+ return capacities
+
+
+def analyze(
+ train,
+ evaluation,
+ specs,
+ budget_bytes,
+ policies,
+ learned_counts=None,
+ train_cache=None,
+ evaluation_cache=None,
+):
+ learned_counts = learned_counts or training_counts(train)
+ train_cache = train_cache or LayerReplayCache(train, specs, learned_counts)
+ evaluation_cache = evaluation_cache or LayerReplayCache(
+ evaluation, specs, learned_counts)
+ uniform = uniform_capacities(specs, budget_bytes)
+ output = {
+ "budget_bytes": budget_bytes,
+ "specs": {str(k): asdict(v) for k, v in sorted(specs.items())},
+ "allocations": {},
+ "results": {},
+ }
+ output["allocations"]["uniform"] = {
+ "capacities": {str(k): v for k, v in sorted(uniform.items())},
+ "resident_bytes": capacity_bytes(uniform, specs),
+ }
+ output["results"]["uniform"] = evaluate(
+ evaluation, uniform, specs, policies, learned_counts, evaluation_cache)
+ for policy in policies:
+ capacities = dynamic_capacities(
+ train, specs, budget_bytes, policy, learned_counts, train_cache)
+ name = f"dynamic-{policy}"
+ output["allocations"][name] = {
+ "capacities": {str(k): v for k, v in sorted(capacities.items())},
+ "resident_bytes": capacity_bytes(capacities, specs),
+ }
+ output["results"][name] = evaluate(
+ evaluation, capacities, specs, [policy], learned_counts, evaluation_cache)
+ return output
+
+
+def print_report(result):
+ print(f"expert_budget={result['budget_bytes']} bytes")
+ print("allocation policy hit_rate misses bytes_read felt_wait_ms evictions rejected seeded")
+ for allocation, policies in result["results"].items():
+ caps = result["allocations"][allocation]["capacities"]
+ resident = result["allocations"][allocation]["resident_bytes"]
+ print(f"capacities[{allocation}]={caps} resident_bytes={resident}")
+ for policy, payload in policies.items():
+ stats = payload["aggregate"]
+ print(f"{allocation:18} {policy:11} {stats['hit_rate']:8.2%} "
+ f"{stats['misses']:7d} {stats['bytes_read']:12d} "
+ f"{stats['felt_wait_us']/1000:12.3f} "
+ f"{stats['evictions']:9d} {stats['rejected_admissions']:8d} "
+ f"{stats['seeded_objects']:6d}")
+
+
+def print_gate(gate):
+ print("decision_gate baseline=uniform/lru "
+ f"min_mean_gain={gate['minimum_category_mean_gain']:.1%} "
+ f"max_regression={gate['maximum_category_regression']:.1%}")
+ print("allocation policy verdict category_mean worst_category aggregate")
+ for candidate in gate["candidates"]:
+ verdict = "PASS" if candidate["pass"] else "FAIL"
+ print(f"{candidate['allocation']:18} {candidate['policy']:11} {verdict:7} "
+ f"{candidate['category_mean_gain']:13.2%} "
+ f"{candidate['worst_category_gain']:14.2%} "
+ f"{candidate['aggregate_gain']:9.2%}")
+
+
+def synthetic_trace(kind: str):
+ specs = {
+ 0: LayerSpec(1_000_000, 1_000_000, 1000.0, 16),
+ 1: LayerSpec(1_000_000, 1_000_000, 4000.0, 16),
+ 2: LayerSpec(2_000_000, 2_000_000, 2000.0, 16),
+ }
+ budget = 8_000_000
+
+ def events(layer, sequence):
+ return [AccessEvent(layer, (expert,), (expert,), None, kind) for expert in sequence]
+
+ hot0 = [0, 1, 0, 2, 0, 1, 0, 3] * 100
+ hot1 = [0, 1, 0, 2, 0, 3, 0, 1, 4, 0, 2, 0] * 100
+ burst2 = sum(([0, 1] * 8 + [cold] for cold in range(2, 12)), []) * 20
+ train_trace = events(0, hot0) + events(1, hot1) + events(2, burst2)
+ if kind == "stationary":
+ eval_trace = events(0, hot0[:400]) + events(1, hot1[:600]) + events(2, burst2[:400])
+ elif kind == "shifted":
+ shifted1 = list(range(16)) * 40
+ shifted0 = [7, 8] * 300
+ eval_trace = events(0, shifted0) + events(1, shifted1) + events(2, burst2[:300])
+ else:
+ raise ValueError(kind)
+ return [train_trace], [eval_trace], specs, budget
+
+
+def synthetic_category_suite():
+ """Two-category transfer check: one stationary and one shifted workload."""
+ stationary_train, stationary_eval, specs, budget = synthetic_trace("stationary")
+ _shifted_train, shifted_eval, _shifted_specs, _shifted_budget = synthetic_trace("shifted")
+ return (
+ stationary_train,
+ [stationary_eval[0], shifted_eval[0]],
+ specs,
+ budget,
+ ["stationary", "shifted"],
+ )
+
+
+def synthetic_budget_sweep(policies, budgets_mb=(2, 4, 6, 8, 10, 12, 16, 24, 32)):
+ train, evaluation, specs, _budget, categories = synthetic_category_suite()
+ rows = []
+ for budget_mb in budgets_mb:
+ budget_bytes = budget_mb * 1_000_000
+ result = analyze(train, evaluation, specs, budget_bytes, policies)
+ gate = decision_gate(result, categories)
+ sensitivity = sensitivity_analysis(
+ train, evaluation, specs, budget_bytes, policies, categories,
+ [0.5, 1.0, 1.5])
+ robust = {
+ (row["allocation"], row["policy"]): row["pass_all_scales"]
+ for row in sensitivity["candidates"]
+ }
+ for candidate in gate["candidates"]:
+ if candidate["pass"]:
+ rows.append({
+ "budget_mb": budget_mb,
+ "allocation": candidate["allocation"],
+ "policy": candidate["policy"],
+ "category_mean_gain": candidate["category_mean_gain"],
+ "worst_category_gain": candidate["worst_category_gain"],
+ "pass_sensitivity": robust[
+ (candidate["allocation"], candidate["policy"])],
+ })
+ return rows
+
+
+def command_demo(args):
+ for scenario in ("stationary", "shifted"):
+ train, evaluation, specs, budget = synthetic_trace(scenario)
+ print(f"\n=== {scenario} ===")
+ result = analyze(train, evaluation, specs, budget, args.policies)
+ print_report(result)
+ if "lru" in args.policies:
+ print_gate(decision_gate(result, [scenario]))
+ if "lru" in args.policies:
+ train, evaluation, specs, budget, categories = synthetic_category_suite()
+ print("\n=== combined category-held-out gate ===")
+ result = analyze(train, evaluation, specs, budget, args.policies)
+ print_gate(decision_gate(result, categories))
+ sensitivity = sensitivity_analysis(
+ train, evaluation, specs, budget, args.policies, categories,
+ [0.5, 1.0, 1.5])
+ print("sensitivity " + " ".join(
+ f"{row['allocation']}/{row['policy']}="
+ f"{'PASS' if row['pass_all_scales'] else 'FAIL'}"
+ for row in sensitivity["candidates"]))
+ sweep = synthetic_budget_sweep(args.policies)
+ print("\n=== synthetic budget operating regions ===")
+ print("budget_mb allocation policy category_mean worst_category sensitivity")
+ for row in sweep:
+ print(f"{row['budget_mb']:9d} {row['allocation']:18} {row['policy']:11} "
+ f"{row['category_mean_gain']:13.2%} "
+ f"{row['worst_category_gain']:14.2%} "
+ f"{'PASS' if row['pass_sensitivity'] else 'FAIL'}")
+
+
+def _real_inputs(args):
+ train_paths = [path.resolve() for path in args.train]
+ eval_paths = [path.resolve() for path in args.eval]
+ if len(set(train_paths)) != len(train_paths):
+ raise ValueError("duplicate path in --train")
+ if len(set(eval_paths)) != len(eval_paths):
+ raise ValueError("duplicate path in --eval")
+ overlap = set(train_paths) & set(eval_paths)
+ if overlap:
+ raise ValueError(f"train/eval paths overlap: {sorted(map(str, overlap))[0]}")
+ try:
+ train_identities = [(path.stat().st_dev, path.stat().st_ino) for path in train_paths]
+ eval_identities = [(path.stat().st_dev, path.stat().st_ino) for path in eval_paths]
+ except OSError:
+ train_identities = []
+ eval_identities = []
+ if len(set(train_identities)) != len(train_identities):
+ raise ValueError("duplicate file in --train")
+ if len(set(eval_identities)) != len(eval_identities):
+ raise ValueError("duplicate file in --eval")
+ if set(train_identities) & set(eval_identities):
+ raise ValueError("train/eval paths refer to the same file")
+ train = read_traces(train_paths)
+ evaluation = read_traces(eval_paths)
+ specs = load_specs(train + evaluation, args.manifest,
+ args.read_gbps, args.felt_fraction)
+ return train, evaluation, specs
+
+
+def command_run(args):
+ train, evaluation, specs = _real_inputs(args)
+ if args.max_training_events:
+ train = [trace[:args.max_training_events] for trace in train]
+ if any(not trace for trace in train):
+ raise ValueError("--max-training-events removed every event from a trace")
+ budget = int(args.expert_budget_gb * 1e9)
+ result = analyze(train, evaluation, specs, budget, args.policies)
+ print_report(result)
+ categories = trace_categories(evaluation, args.eval_category)
+ gate = decision_gate(result, categories, args.minimum_gain, args.maximum_regression)
+ print_gate(gate)
+ sensitivity = sensitivity_analysis(
+ train, evaluation, specs, budget, args.policies, categories,
+ args.felt_scales, args.minimum_gain, args.maximum_regression,
+ args.sensitivity_layers)
+ costs = profile_costs(args.prof_log)
+ print("sensitivity " + " ".join(
+ f"{row['allocation']}/{row['policy']}="
+ f"{'PASS' if row['pass_all_scales'] else 'FAIL'}"
+ for row in sensitivity["candidates"]))
+ if args.json_out:
+ document = {
+ "result": result,
+ "categories": categories,
+ "profile_costs": costs,
+ "gate": gate,
+ "sensitivity": sensitivity,
+ }
+ args.json_out.write_text(json.dumps(document, indent=2) + "\n", encoding="utf-8")
+
+
+def command_sweep(args):
+ train, evaluation, specs = _real_inputs(args)
+ if args.max_training_events:
+ train = [trace[:args.max_training_events] for trace in train]
+ if any(not trace for trace in train):
+ raise ValueError("--max-training-events removed every event from a trace")
+ categories = trace_categories(evaluation, args.eval_category)
+ costs = profile_costs(args.prof_log)
+ results = []
+ for budget_gb in args.expert_budgets:
+ result = analyze(train, evaluation, specs, int(budget_gb * 1e9), args.policies)
+ results.append({
+ "expert_budget_gb": budget_gb,
+ "result": result,
+ "profile_costs": costs,
+ "gate": decision_gate(
+ result, categories, args.minimum_gain, args.maximum_regression),
+ "sensitivity": sensitivity_analysis(
+ train, evaluation, specs, int(budget_gb * 1e9), args.policies,
+ categories, args.felt_scales, args.minimum_gain,
+ args.maximum_regression, args.sensitivity_layers),
+ })
+ for entry in results:
+ print(f"\n=== expert budget {entry['expert_budget_gb']:g} GB ===")
+ print_report(entry["result"])
+ print_gate(entry["gate"])
+ print("sensitivity " + " ".join(
+ f"{row['allocation']}/{row['policy']}="
+ f"{'PASS' if row['pass_all_scales'] else 'FAIL'}"
+ for row in entry["sensitivity"]["candidates"]))
+ if args.json_out:
+ args.json_out.write_text(json.dumps({"sweeps": results}, indent=2) + "\n",
+ encoding="utf-8")
+
+
+def add_real_trace_arguments(parser):
+ parser.add_argument("--train", type=Path, nargs="+", required=True)
+ parser.add_argument("--eval", type=Path, nargs="+", required=True)
+ parser.add_argument("--manifest", type=Path, required=True)
+ parser.add_argument("--read-gbps", type=float, default=3.0)
+ parser.add_argument("--felt-fraction", type=float, default=1.0)
+ parser.add_argument("--felt-scales", type=float, nargs="+", default=[0.5, 1.0, 1.5])
+ parser.add_argument("--eval-category", action="append")
+ parser.add_argument("--minimum-gain", type=float, default=0.10)
+ parser.add_argument("--maximum-regression", type=float, default=0.03)
+ parser.add_argument("--max-training-events", type=int, default=0,
+ help="limit events per training trace for a quick smoke run")
+ parser.add_argument("--sensitivity-layer", dest="sensitivity_layers",
+ type=int, action="append",
+ help="limit cost sensitivity to selected layers")
+ parser.add_argument("--prof-log", type=Path, action="append", default=[],
+ help="GLM PROF=1 log used for aggregate cost calibration")
+ parser.add_argument("--policies", nargs="+", choices=tuple(POLICIES),
+ default=list(POLICIES))
+ parser.add_argument("--json-out", type=Path)
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ sub = parser.add_subparsers(dest="command", required=True)
+
+ demo = sub.add_parser("demo", help="run deterministic stationary/shifted fixtures")
+ demo.add_argument("--policies", nargs="+", choices=tuple(POLICIES),
+ default=list(POLICIES))
+ demo.set_defaults(func=command_demo)
+
+ run = sub.add_parser("run", help="simulate real GLM ROUTE_TRACE files")
+ add_real_trace_arguments(run)
+ run.add_argument("--expert-budget-gb", type=float, required=True)
+ run.set_defaults(func=command_run)
+
+ sweep = sub.add_parser("sweep", help="compare policies over expert RAM budgets")
+ add_real_trace_arguments(sweep)
+ sweep.add_argument("--expert-budgets", type=float, nargs="+", required=True)
+ sweep.set_defaults(func=command_sweep)
+ args = parser.parse_args()
+ budgets = ([args.expert_budget_gb] if hasattr(args, "expert_budget_gb")
+ else getattr(args, "expert_budgets", []))
+ if any(value <= 0 or not math.isfinite(value) or value > 1e9 for value in budgets):
+ parser.error("expert budget values must be positive, finite, and <= 1e9 GB")
+ if hasattr(args, "felt_scales") and any(
+ value <= 0 or not math.isfinite(value) for value in args.felt_scales):
+ parser.error("--felt-scales values must be positive and finite")
+ if hasattr(args, "minimum_gain") and not 0 <= args.minimum_gain < 1:
+ parser.error("--minimum-gain must be in [0,1)")
+ if hasattr(args, "maximum_regression") and args.maximum_regression < 0:
+ parser.error("--maximum-regression must be non-negative")
+ if hasattr(args, "max_training_events") and args.max_training_events < 0:
+ parser.error("--max-training-events must be non-negative")
+ if hasattr(args, "policies") and "lru" not in args.policies:
+ parser.error("--policies must include lru as the decision-gate baseline")
+ try:
+ args.func(args)
+ except (OSError, ValueError, json.JSONDecodeError) as error:
+ parser.error(str(error))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/docs/experiments/cnre-offline-simulator.md b/docs/experiments/cnre-offline-simulator.md
new file mode 100644
index 00000000..3f65da31
--- /dev/null
+++ b/docs/experiments/cnre-offline-simulator.md
@@ -0,0 +1,281 @@
+# CNRE Phase 0: offline residency-policy simulator
+
+This experiment supports [Discussion #884](https://github.com/JustVugg/colibri/discussions/884).
+It does not modify an engine or claim a runtime speedup. It replays existing
+`ROUTE_TRACE` files through bounded expert caches so weak policies can be
+rejected before changing inference code or renting hardware.
+
+## Scope
+
+`c/tools/residency_sim.py` currently models the main GLM CPU-routing path:
+
+- first-seen expert unions for each contiguous `(call, layer)` batch;
+- promotion between GLM's 64-expert working-set blocks;
+- separate per-layer resident footprint, read bytes, and felt miss cost;
+- a fixed expert-residency RAM budget, excluding the rest of process RAM;
+- equal slots per layer versus policy-specific, trace-trained layer budgets;
+- GLM LRU, synthetic half-capacity pins + LRU, decayed LFU, segmented LRU,
+ and decayed-frequency admission.
+
+The half-pinned and alternative admission policies are experiments, not exact
+reproductions of current GLM startup pins or live `REPIN`.
+
+The tool does **not** model new prefetch reads, routing substitutions, GPU
+kernels, NUMA effects, page-cache interference, lock contention, startup read
+cost, or actual tok/s. A predicted reduction in felt wait is a gate for a
+runtime A/B, not the result of one.
+
+Current GLM also read-aheads the next 64-expert block while computing the
+current block. The simulator preserves block promotion semantics but does not
+model this existing overlap. Long-prefill felt-wait estimates are therefore
+conservative and must be checked with lower `felt_fraction` sensitivity.
+
+Kimi K3 traces are not currently safe input because that engine emits routes
+without advancing the trace call ID. Inkling and OLMoE initialize routing
+telemetry but do not currently emit route records. Their cache traversal also
+differs from GLM, so each needs an explicit engine profile before inclusion.
+
+## Trace contract
+
+The input is the engine's current `ROUTE_TRACE` stream:
+
+```text
+<call> <row> <layer> <expert>:<gate> ...
+```
+
+Rows sharing one contiguous `(call, layer)` group are one runtime batch. The
+simulator unions their expert IDs in first-seen order for demand I/O, while
+retaining per-row selection multiplicity to train startup residents like
+`.coli_usage` counts. It rejects malformed fields, missing/reordered rows,
+non-advancing call IDs, and repeated noncontiguous call groups rather than
+silently merging a broken trace.
+
+Use a clean environment and deterministic, placement-neutral GLM trace
+collection. In particular, unset `EXPERT_BUDGET` and `ABLATE_*`; the former is
+applied after tracing and would make the trace differ from executed demand.
+Record `COLI_PREFILL_CHUNK`, because it changes batch boundaries.
+
+For a large model, place the model and traces on the instance's local scratch
+volume rather than the root disk. On DigitalOcean's H200 image this is commonly
+`/dev/vdc1`, mounted at `/mnt/scratch`; verify with `lsblk` and `findmnt` before
+using it. The scratch volume is ephemeral and must not be treated as durable
+storage.
+
+```sh
+env -i PATH="$PATH" HOME="$HOME" \
+ DRAFT=0 MTP=0 TOPP=0 TOPK=0 CACHE_ROUTE=0 \
+ PILOT=0 PILOT_REAL=0 REPIN=0 TEMP=0 \
+ ROUTE_TRACE=trace.txt \
+ SNAP=/path/to/model PROMPT="..." NGEN=128 ./colibri 64 4 8
+```
+
+Training traces and held-out evaluation traces must be separate files. The CLI
+rejects identical resolved paths. Use one file per cold process run; policy
+state is intentionally reset between files.
+
+## Manifest and cost calibration
+
+A manifest with fixed model geometry is required. Inferring `max_experts` from
+held-out IDs would leak evaluation data and understate dimensions when a valid
+expert did not happen to route.
+
+```json
+{
+ "layers": {
+ "3": {
+ "resident_bytes": 18915328,
+ "read_bytes": 75661312,
+ "felt_miss_us": 2400.0,
+ "max_experts": 256
+ }
+ }
+}
+```
+
+Manifest rows that are absent from all supplied traces are ignored. This keeps
+disabled MTP rows from consuming only the uniform baseline's budget.
+
+`resident_bytes` consumes the expert cache budget. `read_bytes` counts demand
+I/O and may differ when a source tensor is transformed into a smaller resident
+representation. If `felt_miss_us` is omitted, it defaults to:
+
+```text
+read_bytes / (read_gbps * 1e9) * 1e6 * felt_fraction
+```
+
+`felt_fraction` is the fraction of read service not hidden by concurrent
+compute. Caching a 20 ms read that is fully overlapped does not remove 20 ms
+from token latency.
+
+## Commands
+
+Run deterministic fixtures:
+
+```sh
+cd c
+python3 tools/residency_sim.py demo
+python3 -m unittest tests.test_residency_sim
+```
+
+Evaluate a fixed expert-cache budget:
+
+```sh
+python3 tools/residency_sim.py run \
+ --train traces/chat_train.trace traces/code_train.trace \
+ --eval traces/chat_heldout_1.trace traces/code_heldout_1.trace \
+ --eval-category chat --eval-category code \
+ --manifest glm-residency.json \
+ --expert-budget-gb 32 \
+ --read-gbps 3.0 \
+ --felt-fraction 0.7 \
+ --felt-scales 0.5 1.0 1.5 \
+ --sensitivity-layer 3 --sensitivity-layer 30 --sensitivity-layer 60 \
+ --prof-log /mnt/scratch/logs/trace/coding_eval.out \
+ --json-out cnre-32gb.json
+```
+
+Sweep expert-cache capacities:
+
+```sh
+python3 tools/residency_sim.py sweep \
+ --train traces/chat_train.trace traces/code_train.trace \
+ --eval traces/chat_heldout_1.trace traces/code_heldout_1.trace \
+ --eval-category chat --eval-category code \
+ --manifest glm-residency.json \
+ --expert-budgets 8 16 24 32 48 64 \
+ --read-gbps 3.0 \
+ --felt-fraction 0.7 \
+ --json-out cnre-sweep.json
+```
+
+JSON preserves model specs, every allocation, per-trace source paths and
+metrics, and each complete sweep result. This is enough to compute category
+regressions outside the simulator without parsing human output.
+
+`--policies` must include `lru`, because every decision is measured against the
+uniform-LRU baseline. Both `run` and every point in `sweep` report nominal and
+layer-cost-sensitivity verdicts.
+
+`--prof-log` stores the aggregate `PROF=1` felt-wait calibration in the JSON.
+Current GLM logs expose aggregate, not per-layer, felt wait; this is provenance
+and a global calibration check, not a claim of layer-specific measurement.
+
+`run` also applies the Phase-0 decision gate directly. Categories receive equal
+weight regardless of their number or length of traces. A candidate passes only
+when its category-mean felt-wait gain is at least 10% and its worst category is
+not more than 3% slower than uniform LRU. If `--eval-category` is omitted, the
+tool derives categories by stripping a final numeric replicate from file names,
+for example `chat_2.trace` -> `chat`.
+
+Sensitivity reruns the complete training allocation and held-out evaluation
+after scaling each layer's felt miss cost independently. `--felt-scales 0.5 1
+1.5` therefore tests the baseline plus low/high cost perturbations for every
+active layer. Uniformly scaling every layer would leave policy rankings and
+percentage gains unchanged and is not a useful robustness test.
+
+For a large model, use a representative cost sample first, for example
+`--sensitivity-layer 3 --sensitivity-layer 30 --sensitivity-layer 60`. The
+default all-layer sensitivity is intentionally thorough but can be expensive
+because it recomputes policy-specific capacity allocation for every perturbation.
+
+## Deterministic fixtures
+
+The built-in stationary workload is intentionally cacheable and demonstrates
+scan resistance. The shifted held-out workload deliberately changes its hot
+set and demonstrates that training-seeded residents and dynamic layer budgets
+can overfit. Exact fixture numbers are printed by `demo` and asserted only where
+they encode an invariant; they are not evidence about real Colibri workloads.
+
+`demo` then evaluates both fixtures as equal-weight categories in one decision
+gate. This is deliberately stricter than averaging all accesses: a large
+stationary win cannot hide a shifted-category regression beyond 3%. In the
+current fixture, uniform frequency admission fails that guard (`-4.76%` in the
+shifted category). Dynamic frequency admission narrowly passes the nominal
+gate (`+4.27%` worst category) but fails the layer-by-layer felt-cost
+sensitivity check at that 8 MB budget.
+
+The final demo table sweeps synthetic expert budgets from 2 to 32 MB. It shows
+that robustness is an operating region rather than a policy-wide property. For
+example, dynamic frequency admission is robust in the deliberately constrained
+2-6 MB fixture, fragile at 8-12 MB, and robust again at larger toy budgets. These
+transitions are artifacts of the fixture, but they demonstrate why real traces
+must be swept across realistic RAM budgets rather than evaluated at one point.
+
+## Phase-0 decision gate
+
+Advance to runtime telemetry or an opt-in cache policy only if category-held-out
+real traces show:
+
+```text
+at least 10% lower predicted felt wait
+no principal held-out category worse by more than 3%
+bounded churn and exact expert-cache byte-budget compliance
+the result survives conservative miss-cost sensitivity
+```
+
+The expert-cache budget does not establish whole-process RAM compliance. A
+runtime implementation must additionally account for dense weights, KV state,
+working-set slabs, scratch, page cache, and OS reserve. A negative result is a
+completed experiment and should be published.
+
+## Data still needed
+
+The repository does not currently contain complete GLM routing traces. The
+first useful campaign needs:
+
+1. GLM-5.2 training and held-out traces from coding, chat, multilingual,
+ reasoning, and long-context prompts.
+2. Matching `PROF=1` logs or a per-layer felt-cost manifest.
+3. Exact commit, model/container identity, DRAFT/sampling settings, cache state,
+ prefill chunk, and frozen `.coli_usage` snapshot.
+4. Separate instrumentation fixes and engine profiles before Kimi K3, Inkling,
+ or OLMoE data is interpreted by this simulator.
+
+No server is required to run the simulator. A model-capable machine is needed
+only to collect missing traces and later validate a policy end to end.
+
+## H200 pilot result
+
+The first real-trace campaign ran on a DigitalOcean H200 instance using the
+gs64 GLM container at revision
+`95bb5f03e3e0ca16b4c711394d0461fa86a3cfcb`. The model occupied 429.3 GB on the
+5 TB local scratch volume. The machine had 24 physical CPU cores, 247 GB RAM,
+and 143.8 GiB H200 VRAM.
+
+The collection produced eight training traces and five held-out traces across
+coding, chat, reasoning, multilingual, and long-context prompts. These are
+small pilot traces, not a production benchmark corpus.
+
+At expert-cache budgets of 120, 160, and 200 GB, the offline simulator reported:
+
+```text
+budget candidate mean held-out gain worst category
+120 GB uniform frequency +39.91% +34.84%
+120 GB dynamic frequency +40.45% +35.41%
+160 GB uniform frequency +53.35% +49.92%
+160 GB dynamic frequency +54.50% +50.66%
+200 GB uniform frequency +65.31% +62.64%
+200 GB dynamic frequency +66.29% +63.07%
+```
+
+All frequency candidates passed the nominal category gate and the sampled
+layer-cost sensitivity at layers 3, 30, and 60. LRU dynamic allocation did not
+meet the 10% gate, with approximately 0.0-0.4% gain.
+
+The runtime pilot is the necessary qualification. With `RAM_GB=120`, `PIPE=0`
+measured 1.66 tok/s at 57.2% hit; with `RAM_GB=200`, it measured 1.31 tok/s at
+73.5% hit. Enabling the existing `PIPE=1` overlap reached 2.00 tok/s at the
+same 57.2% hit. This means hit rate alone is not a performance result: I/O
+overlap, disk service, and compute contention materially change tok/s.
+
+Across the five held-out `PROF=1` logs, the aggregate runtime instrumentation
+reported 19.6 seconds of felt wait over 2,625 loads, or approximately 7.47 ms
+per load. This is a whole-run calibration signal, not a per-layer cost: the
+current GLM profile does not attribute felt wait to individual layers.
+
+No runtime frequency-admission policy was implemented in this pilot. The
+current GLM cache admits every demand miss in `moe()` and promotes the bounded
+tail of each 64-expert block. An unrecognized `COLI_CACHE_ADMISSION` variable
+therefore has no effect. This is intentional: the real-trace evidence is strong
+enough to justify a narrowly scoped opt-in prototype, but not enough to claim a
+runtime win or to make a misleading A/B number part of the RFC.