Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
experiment: add offline residency simulator
Mario Gutierrezcc18bfbb3f44
diff --git a/c/tests/test_residency_sim.py b/c/tests/test_residency_sim.pynew file mode 100644index 00000000..4967e729--- /dev/null+++ b/c/tests/test_residency_sim.py@@ -0,0 +1,351 @@+import json+import sys+import tempfile+import unittest+from pathlib import Path+++TOOLS = Path(__file__).resolve().parent.parent / "tools"+sys.path.insert(0, str(TOOLS))+import residency_sim as sim+++class TraceParserTest(unittest.TestCase):+ def test_batch_rows_keep_union_order_and_selection_multiplicity(self):+ with tempfile.TemporaryDirectory() as tmp:+ path = Path(tmp) / "trace.txt"+ path.write_text(+ "0 0 3 7:0.6000 2:0.4000\n"+ "0 1 3 2:0.7000 9:0.3000\n"+ "1 0 4 1:1.0000\n",+ encoding="utf-8",+ )+ events = sim.parse_trace(path)+ self.assertEqual(events, [+ sim.AccessEvent(3, (7, 2, 9), (7, 2, 2, 9), 0, str(path)),+ sim.AccessEvent(4, (1,), (1,), 1, str(path)),+ ])++ def test_multiple_calls_to_same_layer_remain_separate_events(self):+ with tempfile.TemporaryDirectory() as tmp:+ path = Path(tmp) / "trace.txt"+ path.write_text("0 0 3 1:1.0000\n1 0 3 2:1.0000\n", encoding="utf-8")+ events = sim.parse_trace(path)+ self.assertEqual([event.experts for event in events], [(1,), (2,)])++ def test_noncontiguous_group_is_rejected(self):+ with tempfile.TemporaryDirectory() as tmp:+ path = Path(tmp) / "trace.txt"+ path.write_text(+ "0 0 3 1:1.0000\n1 0 4 2:1.0000\n0 1 3 3:1.0000\n",+ encoding="utf-8",+ )+ with self.assertRaisesRegex(ValueError, "noncontiguous"):+ sim.parse_trace(path)++ def test_kimi_style_constant_call_ids_are_rejected(self):+ events = [+ sim.AccessEvent(0, (1,), call=0),+ sim.AccessEvent(1, (2,), call=0),+ sim.AccessEvent(2, (3,), call=0),+ ]+ with self.assertRaisesRegex(ValueError, "lacks advancing GLM call ids"):+ sim.validate_glm_trace(events, Path("kimi.trace"))++ def test_glm_call_ids_advance_across_layer_wraps(self):+ events = [+ sim.AccessEvent(0, (1,), call=0),+ sim.AccessEvent(1, (2,), call=1),+ sim.AccessEvent(2, (3,), call=2),+ sim.AccessEvent(0, (4,), call=3),+ sim.AccessEvent(1, (5,), call=4),+ ]+ sim.validate_glm_trace(events, Path("glm.trace"))++ def test_glm_call_ids_must_start_at_zero_and_advance_at_wrap(self):+ invalid = (+ [sim.AccessEvent(0, (1,), call=1)],+ [sim.AccessEvent(0, (1,), call=0), sim.AccessEvent(1, (2,), call=1),+ sim.AccessEvent(0, (3,), call=1)],+ )+ for events in invalid:+ with self.subTest(events=events), self.assertRaisesRegex(+ ValueError, "lacks advancing GLM call ids"):+ sim.validate_glm_trace(events, Path("glm.trace"))++ def test_rows_must_start_at_zero_and_be_contiguous(self):+ invalid = (+ "0 1 3 7:1.0000\n",+ "0 0 3 7:1.0000\n0 2 3 8:1.0000\n",+ )+ for text in invalid:+ with self.subTest(text=text), tempfile.TemporaryDirectory() as tmp:+ path = Path(tmp) / "trace.txt"+ path.write_text(text, encoding="utf-8")+ with self.assertRaisesRegex(ValueError, "expected row"):+ sim.parse_trace(path)++ def test_malformed_or_nonfinite_gate_is_rejected_with_location(self):+ invalid = ("0 0 3 bad\n", "0 0 3 7:nan\n", "0 0 3 7:1:2\n")+ for text in invalid:+ with self.subTest(text=text), tempfile.TemporaryDirectory() as tmp:+ path = Path(tmp) / "trace.txt"+ path.write_text(text, encoding="utf-8")+ with self.assertRaisesRegex(ValueError, r"trace\.txt:1"):+ sim.parse_trace(path)+++class SpecificationTest(unittest.TestCase):+ def test_manifest_separates_resident_and_read_bytes(self):+ traces = [[sim.AccessEvent(3, (0, 1))]]+ with tempfile.TemporaryDirectory() as tmp:+ path = Path(tmp) / "manifest.json"+ path.write_text(json.dumps({"layers": {"3": {+ "resident_bytes": 123,+ "read_bytes": 456,+ "felt_miss_us": 45.5,+ "max_experts": 9,+ }}}), encoding="utf-8")+ specs = sim.load_specs(traces, path, 1.0, 1.0)+ self.assertEqual(specs[3], sim.LayerSpec(123, 456, 45.5, 9))++ def test_manifest_bounds_are_required_for_observed_layers_and_experts(self):+ traces = [[sim.AccessEvent(3, (9,))]]+ with tempfile.TemporaryDirectory() as tmp:+ path = Path(tmp) / "manifest.json"+ path.write_text(json.dumps({"layers": {"3": {+ "resident_bytes": 123,+ "max_experts": 9,+ }}}), encoding="utf-8")+ with self.assertRaisesRegex(ValueError, "exceeds manifest bound"):+ sim.load_specs(traces, path, 1.0, 1.0)++ def test_default_felt_cost_uses_read_bytes(self):+ traces = [[sim.AccessEvent(0, (0,))]]+ with tempfile.TemporaryDirectory() as tmp:+ path = Path(tmp) / "manifest.json"+ path.write_text(json.dumps({"layers": {"0": {+ "resident_bytes": 10,+ "read_bytes": 1_000_000_000,+ "max_experts": 1,+ }}}), encoding="utf-8")+ specs = sim.load_specs(traces, path, 2.0, 0.5)+ self.assertEqual(specs[0].felt_miss_us, 250_000)++ def test_manifest_layers_absent_from_traces_do_not_consume_budget(self):+ traces = [[sim.AccessEvent(0, (0,))]]+ with tempfile.TemporaryDirectory() as tmp:+ path = Path(tmp) / "manifest.json"+ path.write_text(json.dumps({"layers": {+ "0": {"resident_bytes": 10, "max_experts": 1},+ "1": {"resident_bytes": 10, "max_experts": 1},+ }}), encoding="utf-8")+ specs = sim.load_specs(traces, path, 1.0, 1.0)+ self.assertEqual(set(specs), {0})++ def test_profile_costs_extracts_aggregate_felt_wait(self):+ with tempfile.TemporaryDirectory() as tmp:+ path = Path(tmp) / "profile.out"+ path.write_text(+ "[PROF] expert I/O: 38.178 GB fetched | 525.0 loads/token | "+ "7.2s read service / 1.4s felt wait\n",+ encoding="utf-8",+ )+ costs = sim.profile_costs([path])+ self.assertEqual(costs["loads"], 525)+ self.assertEqual(costs["felt_wait_seconds"], 1.4)+ self.assertAlmostEqual(costs["felt_us_per_load"], 2666.6666667)+++class PolicyBehaviorTest(unittest.TestCase):+ def specs(self):+ return {0: sim.LayerSpec(1, 4, 10, 100)}++ def events(self, sequence):+ return [sim.AccessEvent(0, (expert,), (expert,)) for expert in sequence]++ def test_lru_known_sequence_and_read_bytes(self):+ stats = sim.run_policy("lru", {0: 2}, self.specs(),+ self.events([1, 2, 1, 3, 1, 2]))+ self.assertEqual((stats.hits, stats.misses), (2, 4))+ self.assertEqual(stats.bytes_read, 16)+ self.assertEqual(stats.felt_wait_us, 40)++ def test_frequency_admission_rejects_scan_pollution(self):+ warm = [0, 1] * 20+ sequence = warm[:]+ for cold in range(2, 22):+ sequence.extend([cold, 0, 1])+ lru = sim.run_policy("lru", {0: 2}, self.specs(), self.events(sequence))+ frequency = sim.run_policy(+ "frequency", {0: 2}, self.specs(), self.events(sequence))+ self.assertGreater(frequency.hits, lru.hits)+ self.assertLess(frequency.misses, lru.misses)+ self.assertGreater(frequency.rejected_admissions, 0)++ def test_training_uses_row_selection_multiplicity(self):+ event = sim.AccessEvent(0, (7, 8), (7, 7, 7, 8))+ counts = sim.training_counts([[event]])+ self.assertEqual(counts[0], {7: 3, 8: 1})++ def test_training_counts_seed_frequency_and_half_pinned_policies(self):+ learned = {0: {7: 100, 8: 50, 9: 1}}+ first = self.events([7, 8])+ frequency = sim.run_policy(+ "frequency", {0: 2}, self.specs(), first, learned)+ pinned = sim.run_policy(+ "half-pinned", {0: 2}, self.specs(), first, learned)+ self.assertEqual(frequency.hits, 2)+ self.assertEqual(frequency.seeded_objects, 2)+ self.assertEqual(pinned.hits, 1)+ self.assertEqual(pinned.seeded_objects, 1)++ def test_slru_protects_reused_objects_from_one_hit_scan(self):+ warm = [0, 1] * 20+ sequence = warm + sum(([cold, 0, 1] for cold in range(2, 22)), [])+ lru = sim.run_policy("lru", {0: 3}, self.specs(), self.events(sequence))+ slru = sim.run_policy("slru", {0: 3}, self.specs(), self.events(sequence))+ self.assertGreaterEqual(slru.hits, lru.hits)++ def test_lru_promotes_between_64_expert_blocks(self):+ event = sim.AccessEvent(0, tuple(range(65)) + (0,))+ stats = sim.run_policy("lru", {0: 64}, self.specs(), [event])+ self.assertEqual(stats.hits, 1)+ self.assertEqual(stats.misses, 65)+++class AllocationTest(unittest.TestCase):+ def test_uniform_allocation_respects_resident_bytes(self):+ specs = {+ 0: sim.LayerSpec(10, 100, 1, 8),+ 1: sim.LayerSpec(20, 100, 1, 8),+ }+ caps = sim.uniform_capacities(specs, 95)+ self.assertEqual(caps, {0: 3, 1: 3})+ self.assertEqual(sim.capacity_bytes(caps, specs), 90)++ def test_dynamic_budget_uses_exact_policy_replay(self):+ specs = {+ 0: sim.LayerSpec(1, 1, 1, 8),+ 1: sim.LayerSpec(1, 1, 10, 8),+ }+ trace = [[+ *[sim.AccessEvent(0, (expert,)) for expert in [0, 1, 2, 3] * 20],+ *[sim.AccessEvent(1, (expert,)) for expert in [0, 1] * 40],+ ]]+ counts = sim.training_counts(trace)+ caps = sim.dynamic_capacities(trace, specs, 4, "lru", counts)+ self.assertGreater(caps[1], caps[0])+ self.assertLessEqual(sim.capacity_bytes(caps, specs), 4)++ def test_training_traces_are_independent_cold_runs(self):+ specs = {0: sim.LayerSpec(1, 1, 1, 4)}+ traces = [+ [sim.AccessEvent(0, (1,))],+ [sim.AccessEvent(0, (1,))],+ ]+ wait = sim._training_wait(traces, {0: 0}, specs, "lru", {})+ self.assertEqual(wait, 2)++ def test_shifted_heldout_dynamic_lru_regression_is_visible(self):+ train, evaluation, specs, budget = sim.synthetic_trace("shifted")+ result = sim.analyze(train, evaluation, specs, budget, ["lru"])+ uniform = result["results"]["uniform"]["lru"]["aggregate"]+ dynamic = result["results"]["dynamic-lru"]["lru"]["aggregate"]+ self.assertGreater(dynamic["felt_wait_us"], uniform["felt_wait_us"])+++class DecisionGateTest(unittest.TestCase):+ def result(self, category_waits):+ def payload(values):+ total = sum(values)+ return {+ "aggregate": {"felt_wait_us": total},+ "traces": [{"felt_wait_us": value} for value in values],+ }++ return {"results": {+ "uniform": {+ "lru": payload([100, 100, 100]),+ "frequency": payload(category_waits),+ },+ "dynamic-frequency": {+ "frequency": payload([70, 70, 70]),+ },+ }}++ def test_gate_equal_weights_categories_and_rejects_worst_regression(self):+ categories = ["chat", "chat", "code"]+ gate = sim.decision_gate(self.result([80, 80, 104]), categories)+ frequency = next(+ row for row in gate["candidates"]+ if row["allocation"] == "uniform" and row["policy"] == "frequency")+ self.assertAlmostEqual(frequency["category_mean_gain"], 0.08)+ self.assertAlmostEqual(frequency["worst_category_gain"], -0.04)+ self.assertFalse(frequency["pass"])+ dynamic = next(+ row for row in gate["candidates"]+ if row["allocation"] == "dynamic-frequency")+ self.assertTrue(dynamic["pass"])++ def test_trace_categories_strip_numeric_replicates_or_use_explicit_names(self):+ traces = [+ [sim.AccessEvent(0, (0,), source="/tmp/chat_1.trace")],+ [sim.AccessEvent(0, (0,), source="/tmp/chat_2.trace")],+ [sim.AccessEvent(0, (0,), source="/tmp/code-1.trace")],+ ]+ self.assertEqual(sim.trace_categories(traces), ["chat", "chat", "code"])+ self.assertEqual(+ sim.trace_categories(traces, ["a", "b", "c"]), ["a", "b", "c"])+ with self.assertRaisesRegex(ValueError, "one value per"):+ sim.trace_categories(traces, ["a"])++ def test_layer_specific_sensitivity_can_invalidate_candidate(self):+ train, evaluation, specs, budget = sim.synthetic_trace("stationary")+ sensitivity = sim.sensitivity_analysis(+ train, evaluation, specs, budget, ["lru", "frequency"],+ ["stationary"], [0.5, 1.0, 1.5])+ frequency = next(+ row for row in sensitivity["candidates"]+ if row["allocation"] == "uniform" and row["policy"] == "frequency")+ self.assertEqual(len(frequency["outcomes"]), 1 + 2 * len(specs))+ self.assertTrue(any(+ outcome["scenario"] == "layer-1-x0.5"+ for outcome in frequency["outcomes"]))++ def test_replay_cache_preserves_full_replay_metrics(self):+ train, evaluation, specs, budget = sim.synthetic_trace("stationary")+ counts = sim.training_counts(train)+ capacities = sim.uniform_capacities(specs, budget)+ cached = sim.evaluate(+ evaluation, capacities, specs, ["lru", "frequency"], counts)+ for policy in ("lru", "frequency"):+ direct = sim.PolicyStats()+ for trace in evaluation:+ direct.add(sim.run_policy(policy, capacities, specs, trace, counts))+ self.assertEqual(cached[policy]["aggregate"], direct.report())++ def test_combined_category_suite_applies_worst_category_guard(self):+ train, evaluation, specs, budget, categories = sim.synthetic_category_suite()+ result = sim.analyze(train, evaluation, specs, budget, ["lru", "frequency"])+ gate = sim.decision_gate(result, categories)+ frequency = next(+ candidate for candidate in gate["candidates"]+ if candidate["allocation"] == "uniform"+ and candidate["policy"] == "frequency")+ self.assertGreater(frequency["category_mean_gain"], 0.10)+ self.assertLess(frequency["worst_category_gain"], -0.03)++ def test_synthetic_budget_sweep_exposes_robust_and_fragile_regions(self):+ rows = sim.synthetic_budget_sweep(["lru", "frequency"], [2, 8])+ frequency = {+ row["budget_mb"]: row+ for row in rows+ if row["allocation"] == "dynamic-frequency"+ }+ self.assertTrue(frequency[2]["pass_sensitivity"])+ self.assertFalse(frequency[8]["pass_sensitivity"])+++if __name__ == "__main__":+ unittest.main()diff --git a/c/tools/residency_sim.py b/c/tools/residency_sim.pynew file mode 100644index 00000000..a4e8a822--- /dev/null+++ b/c/tools/residency_sim.py@@ -0,0 +1,1177 @@+#!/usr/bin/env python3+"""Offline GLM expert-residency policy simulator.++The simulator consumes Colibri ROUTE_TRACE output without loading a model. It+replays GLM's routed-expert unions through bounded per-layer caches and compares+placement policies under the same expert-residency byte budget. It deliberately+models demand residency only: no speculative reads, routing changes, or token+semantics are involved.++ROUTE_TRACE lines have this form (one row per token position and MoE call):++ <call> <row> <layer> <expert>:<gate> ...++Rows sharing one contiguous (call, layer) group are a runtime batch. The parser+retains both the first-seen expert union for demand replay and per-row selection+counts for training startup residents.+"""++from __future__ import annotations++import argparse+import json+import math+import re+import statistics+from collections import Counter, OrderedDict, defaultdict+from dataclasses import asdict, dataclass+from pathlib import Path+from typing import Sequence+++EXPERT_FIELD = re.compile(r"^(\d+):([^:]+)$")+++@dataclass(frozen=True)+class AccessEvent:+ layer: int+ experts: tuple[int, ...]+ selections: tuple[int, ...] = ()+ call: int | None = None+ source: str = ""++ def selection_counts(self) -> Counter[int]:+ return Counter(self.selections or self.experts)+++@dataclass(frozen=True)+class LayerSpec:+ resident_bytes: int+ read_bytes: int+ felt_miss_us: float+ max_experts: int+++@dataclass+class PolicyStats:+ requests: int = 0+ hits: int = 0+ misses: int = 0+ bytes_read: int = 0+ felt_wait_us: float = 0.0+ admissions: int = 0+ rejected_admissions: int = 0+ evictions: int = 0+ seeded_objects: int = 0++ def add(self, other: "PolicyStats") -> None:+ for field in self.__dataclass_fields__:+ setattr(self, field, getattr(self, field) + getattr(other, field))++ def report(self) -> dict:+ accesses = self.hits + self.misses+ return {+ **asdict(self),+ "accesses": accesses,+ "hit_rate": self.hits / accesses if accesses else 1.0,+ }+++def _build_event(key, union, selections, source):+ if key is None or not union:+ return None+ return AccessEvent(key[1], tuple(union), tuple(selections), key[0], source)+++def parse_trace(path: Path) -> list[AccessEvent]:+ """Parse contiguous GLM ROUTE_TRACE call groups into demand-union events."""+ events = []+ key = None+ union: list[int] = []+ known: set[int] = set()+ selections: list[int] = []+ seen_keys: set[tuple[int, int]] = set()+ next_row = 0+ last_call = -1+ for lineno, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):+ fields = raw.split()+ if not fields:+ continue+ if len(fields) < 4:+ raise ValueError(f"{path}:{lineno}: expected call row layer expert:gate ...")+ try:+ call, row, layer = map(int, fields[:3])+ except ValueError as error:+ raise ValueError(f"{path}:{lineno}: invalid call/row/layer") from error+ if call < 0 or row < 0 or layer < 0:+ raise ValueError(f"{path}:{lineno}: negative call/row/layer")+ next_key = (call, layer)+ if next_key != key:+ event = _build_event(key, union, selections, str(path))+ if event:+ events.append(event)+ if next_key in seen_keys:+ raise ValueError(f"{path}:{lineno}: noncontiguous call/layer group {next_key}")+ if call < last_call:+ raise ValueError(f"{path}:{lineno}: call id moved backwards")+ seen_keys.add(next_key)+ key = next_key+ union = []+ known = set()+ selections = []+ next_row = 0+ last_call = call+ if row != next_row:+ raise ValueError(+ f"{path}:{lineno}: expected row {next_row} in call/layer group, got {row}")+ next_row += 1+ for value in fields[3:]:+ match = EXPERT_FIELD.fullmatch(value)+ if not match:+ raise ValueError(f"{path}:{lineno}: invalid expert field {value!r}")+ expert = int(match.group(1))+ try:+ gate = float(match.group(2))+ except ValueError as error:+ raise ValueError(f"{path}:{lineno}: invalid gate in {value!r}") from error+ if not math.isfinite(gate):+ raise ValueError(f"{path}:{lineno}: non-finite gate in {value!r}")+ selections.append(expert)+ if expert not in known:+ known.add(expert)+ union.append(expert)+ event = _build_event(key, union, selections, str(path))+ if event:+ events.append(event)+ return events+++def validate_glm_trace(events: Sequence[AccessEvent], path: Path) -> None:+ if not events or any(event.call is None for event in events):+ return+ calls = [event.call for event in events]+ if calls[0] != 0 or any(+ call != previous + 1 for previous, call in zip(calls, calls[1:])):+ raise ValueError(+ f"{path}: trace lacks advancing GLM call ids; unsupported engine or corrupt trace")+++def read_traces(paths: Sequence[Path]) -> list[list[AccessEvent]]:+ traces = [parse_trace(path) for path in paths]+ if not traces or any(not trace for trace in traces):+ raise ValueError("every trace must contain at least one routing event")+ for path, trace in zip(paths, traces):+ validate_glm_trace(trace, path)+ return traces+++def load_specs(+ traces: Sequence[Sequence[AccessEvent]],+ manifest: Path,+ read_gbps: float,+ felt_fraction: float,+) -> dict[int, LayerSpec]:+ if not math.isfinite(read_gbps) or read_gbps <= 0:+ raise ValueError("read GB/s must be positive and finite")+ if not math.isfinite(felt_fraction) or not 0 <= felt_fraction <= 1:+ raise ValueError("felt fraction must be finite and in [0,1]")+ document = json.loads(manifest.read_text(encoding="utf-8"))+ if not isinstance(document, dict) or not isinstance(document.get("layers"), dict):+ raise ValueError("manifest.layers must be an object")+ specs = {}+ for raw_layer, raw_spec in document["layers"].items():+ try:+ layer = int(raw_layer)+ except (TypeError, ValueError) as error:+ raise ValueError(f"invalid manifest layer {raw_layer!r}") from error+ if layer < 0 or not isinstance(raw_spec, dict):+ raise ValueError(f"invalid layer {raw_layer!r} specification")+ try:+ resident_bytes = int(raw_spec["resident_bytes"])+ read_bytes = int(raw_spec.get("read_bytes", resident_bytes))+ max_experts = int(raw_spec["max_experts"])+ default_felt = (read_bytes / (read_gbps * 1e9) * 1e6 * felt_fraction)+ felt = float(raw_spec.get("felt_miss_us", default_felt))+ except (KeyError, TypeError, ValueError) as error:+ raise ValueError(f"invalid layer {layer} specification") from error+ if (resident_bytes <= 0 or read_bytes <= 0 or max_experts <= 0 or+ felt < 0 or not math.isfinite(felt)):+ raise ValueError(f"invalid layer {layer} specification")+ specs[layer] = LayerSpec(resident_bytes, read_bytes, felt, max_experts)+ observed = defaultdict(set)+ for trace in traces:+ for event in trace:+ observed[event.layer].update(event.experts)+ for layer, experts in observed.items():+ if layer not in specs:+ raise ValueError(f"trace layer {layer} is absent from manifest")+ invalid = [expert for expert in experts if expert >= specs[layer].max_experts]+ if invalid:+ raise ValueError(f"trace layer {layer} expert {min(invalid)} exceeds manifest bound")+ return {layer: spec for layer, spec in specs.items() if layer in observed}+++def uniform_capacities(specs: dict[int, LayerSpec], budget_bytes: int) -> dict[int, int]:+ """Allocate equal slot counts per layer, bounded by each layer's expert count."""+ if budget_bytes < 0:+ raise ValueError("budget must be non-negative")+ capacities = {layer: 0 for layer in specs}+ used = 0+ while True:+ cost = sum(spec.resident_bytes for layer, spec in specs.items()+ if capacities[layer] < spec.max_experts)+ if cost <= 0 or used + cost > budget_bytes:+ break+ for layer, spec in specs.items():+ if capacities[layer] < spec.max_experts:+ capacities[layer] += 1+ used += spec.resident_bytes+ return capacities+++class BasePolicy:+ def __init__(self, capacities, specs, learned_counts=None):+ self.capacities = capacities+ self.specs = specs+ self.learned_counts = learned_counts or {}+ self.stats = PolicyStats()++ def lookup(self, layer: int, expert: int) -> bool:+ raise NotImplementedError++ def observe(self, layer: int, expert: int, hit: bool) -> None:+ pass++ def admit_batch(self, layer: int, misses: Sequence[int]) -> None:+ raise NotImplementedError++ def access(self, event: AccessEvent) -> None:+ # GLM resolves/promotes at most 64 experts at a time.+ for start in range(0, len(event.experts), 64):+ misses = []+ spec = self.specs[event.layer]+ for expert in event.experts[start:start + 64]:+ self.stats.requests += 1+ hit = self.lookup(event.layer, expert)+ self.observe(event.layer, expert, hit)+ if hit:+ self.stats.hits += 1+ else:+ self.stats.misses += 1+ self.stats.bytes_read += spec.read_bytes+ self.stats.felt_wait_us += spec.felt_miss_us+ misses.append(expert)+ self.admit_batch(event.layer, misses)+++class LRUPolicy(BasePolicy):+ def __init__(self, capacities, specs, learned_counts=None):+ super().__init__(capacities, specs, learned_counts)+ self.cache: dict[int, OrderedDict[int, None]] = defaultdict(OrderedDict)++ def lookup(self, layer, expert):+ cache = self.cache[layer]+ if expert not in cache:+ return False+ cache.move_to_end(expert)+ return True++ def admit_batch(self, layer, misses):+ capacity = self.capacities.get(layer, 0)+ if capacity <= 0:+ self.stats.rejected_admissions += len(misses)+ return+ selected = list(misses[-capacity:])+ self.stats.rejected_admissions += len(misses) - len(selected)+ cache = self.cache[layer]+ for expert in reversed(selected):+ if expert in cache:+ cache.move_to_end(expert)+ continue+ while len(cache) >= capacity:+ cache.popitem(last=False)+ self.stats.evictions += 1+ cache[expert] = None+ self.stats.admissions += 1+++class LFUPolicy(BasePolicy):+ """Always-admit LFU with per-layer periodic decay and recency ties."""+ def __init__(self, capacities, specs, learned_counts=None,+ decay_interval=4096, decay=0.5):+ super().__init__(capacities, specs, learned_counts)+ self.resident: dict[int, set[int]] = defaultdict(set)+ self.frequency: dict[int, Counter[int]] = defaultdict(Counter)+ self.last: dict[int, dict[int, int]] = defaultdict(dict)+ self.clock: Counter[int] = Counter()+ self.decay_interval = decay_interval+ self.decay = decay+ for layer, capacity in capacities.items():+ counts = Counter(self.learned_counts.get(layer, {}))+ self.frequency[layer].update(counts)+ ranked = sorted(counts, key=lambda expert: (-counts[expert], expert))[:capacity]+ self.resident[layer].update(ranked)+ self.stats.seeded_objects += len(ranked)++ def lookup(self, layer, expert):+ return expert in self.resident[layer]++ def observe(self, layer, expert, hit):+ self.clock[layer] += 1+ if self.decay_interval and self.clock[layer] % self.decay_interval == 0:+ for candidate in list(self.frequency[layer]):+ self.frequency[layer][candidate] *= self.decay+ if self.frequency[layer][candidate] < 0.5:+ del self.frequency[layer][candidate]+ self.frequency[layer][expert] += 1+ self.last[layer][expert] = self.clock[layer]++ def _victim(self, layer):+ return min(self.resident[layer],+ key=lambda expert: (self.frequency[layer][expert],+ self.last[layer].get(expert, 0), expert))++ def admit_batch(self, layer, misses):+ capacity = self.capacities.get(layer, 0)+ if capacity <= 0:+ self.stats.rejected_admissions += len(misses)+ return+ resident = self.resident[layer]+ for expert in misses:+ if expert in resident:+ continue+ if len(resident) >= capacity:+ resident.remove(self._victim(layer))+ self.stats.evictions += 1+ resident.add(expert)+ self.stats.admissions += 1+++class SLRUPolicy(BasePolicy):+ """Segmented LRU: one-hit objects enter probation; reused objects are protected."""+ def __init__(self, capacities, specs, learned_counts=None, protected_fraction=0.5):+ super().__init__(capacities, specs, learned_counts)+ self.protected_fraction = protected_fraction+ self.probation: dict[int, OrderedDict[int, None]] = defaultdict(OrderedDict)+ self.protected: dict[int, OrderedDict[int, None]] = defaultdict(OrderedDict)++ def _limits(self, layer):+ capacity = self.capacities.get(layer, 0)+ if capacity <= 1:+ return capacity, 0+ protected = min(capacity - 1, max(1, int(capacity * self.protected_fraction)))+ return capacity - protected, protected++ def lookup(self, layer, expert):+ protected = self.protected[layer]+ probation = self.probation[layer]+ if expert in protected:+ protected.move_to_end(expert)+ return True+ if expert not in probation:+ return False+ del probation[expert]+ probation_limit, protected_limit = self._limits(layer)+ if protected_limit:+ while len(protected) >= protected_limit:+ demoted, _ = protected.popitem(last=False)+ probation[demoted] = None+ while len(probation) > probation_limit:+ probation.popitem(last=False)+ self.stats.evictions += 1+ protected[expert] = None+ else:+ probation[expert] = None+ return True++ def admit_batch(self, layer, misses):+ capacity = self.capacities.get(layer, 0)+ probation_limit, _protected_limit = self._limits(layer)+ if capacity <= 0:+ self.stats.rejected_admissions += len(misses)+ return+ probation = self.probation[layer]+ protected = self.protected[layer]+ for expert in misses:+ if expert in probation or expert in protected:+ continue+ probation[expert] = None+ self.stats.admissions += 1+ while len(probation) > probation_limit:+ probation.popitem(last=False)+ self.stats.evictions += 1+++class FrequencyAdmissionPolicy(BasePolicy):+ """Decayed-frequency admission: demand may execute without polluting cache."""+ def __init__(self, capacities, specs, learned_counts=None,+ decay_interval=4096, decay=0.5, hysteresis=0.10):+ super().__init__(capacities, specs, learned_counts)+ self.resident: dict[int, set[int]] = defaultdict(set)+ self.frequency: dict[int, Counter[int]] = defaultdict(Counter)+ self.last: dict[int, dict[int, int]] = defaultdict(dict)+ self.clock: Counter[int] = Counter()+ self.decay_interval = decay_interval+ self.decay = decay+ self.hysteresis = hysteresis+ for layer, capacity in capacities.items():+ counts = Counter(self.learned_counts.get(layer, {}))+ self.frequency[layer].update(counts)+ ranked = sorted(counts, key=lambda expert: (-counts[expert], expert))[:capacity]+ self.resident[layer].update(ranked)+ self.stats.seeded_objects += len(ranked)++ def lookup(self, layer, expert):+ return expert in self.resident[layer]++ def observe(self, layer, expert, hit):+ self.clock[layer] += 1+ if self.decay_interval and self.clock[layer] % self.decay_interval == 0:+ for candidate in list(self.frequency[layer]):+ self.frequency[layer][candidate] *= self.decay+ if self.frequency[layer][candidate] < 0.5:+ del self.frequency[layer][candidate]+ self.frequency[layer][expert] += 1+ self.last[layer][expert] = self.clock[layer]++ def _victim(self, layer):+ return min(self.resident[layer],+ key=lambda expert: (self.frequency[layer][expert],+ self.last[layer].get(expert, 0), expert))++ def admit_batch(self, layer, misses):+ capacity = self.capacities.get(layer, 0)+ if capacity <= 0:+ self.stats.rejected_admissions += len(misses)+ return+ resident = self.resident[layer]+ for expert in misses:+ if expert in resident:+ continue+ if len(resident) < capacity:+ resident.add(expert)+ self.stats.admissions += 1+ continue+ victim = self._victim(layer)+ if (self.frequency[layer][expert] <=+ self.frequency[layer][victim] * (1 + self.hysteresis)):+ self.stats.rejected_admissions += 1+ continue+ resident.remove(victim)+ resident.add(expert)+ self.stats.evictions += 1+ self.stats.admissions += 1+++class HalfPinnedLRUPolicy(BasePolicy):+ """Synthetic half-capacity frequency pins plus adaptive LRU."""+ def __init__(self, capacities, specs, learned_counts=None, pin_fraction=0.5):+ super().__init__(capacities, specs, learned_counts)+ self.pins: dict[int, set[int]] = defaultdict(set)+ self.cache: dict[int, OrderedDict[int, None]] = defaultdict(OrderedDict)+ self.adaptive_capacity = {}+ for layer, capacity in capacities.items():+ pin_count = min(capacity, int(capacity * pin_fraction))+ counts = self.learned_counts.get(layer, {})+ ranked = sorted(counts, key=lambda expert: (-counts[expert], expert))[:pin_count]+ self.pins[layer].update(ranked)+ self.adaptive_capacity[layer] = capacity - len(ranked)+ self.stats.seeded_objects += len(ranked)++ def lookup(self, layer, expert):+ if expert in self.pins[layer]:+ return True+ cache = self.cache[layer]+ if expert not in cache:+ return False+ cache.move_to_end(expert)+ return True++ def admit_batch(self, layer, misses):+ capacity = self.adaptive_capacity.get(layer, 0)+ if capacity <= 0:+ self.stats.rejected_admissions += len(misses)+ return+ selected = list(misses[-capacity:])+ self.stats.rejected_admissions += len(misses) - len(selected)+ cache = self.cache[layer]+ for expert in reversed(selected):+ if expert in self.pins[layer] or expert in cache:+ continue+ while len(cache) >= capacity:+ cache.popitem(last=False)+ self.stats.evictions += 1+ cache[expert] = None+ self.stats.admissions += 1+++POLICIES = {+ "lru": LRUPolicy,+ "half-pinned": HalfPinnedLRUPolicy,+ "lfu": LFUPolicy,+ "slru": SLRUPolicy,+ "frequency": FrequencyAdmissionPolicy,+}+++def training_counts(traces):+ counts: dict[int, Counter[int]] = defaultdict(Counter)+ for trace in traces:+ for event in trace:+ counts[event.layer].update(event.selection_counts())+ return dict(counts)+++def run_policy(policy_name, capacities, specs, trace, learned_counts=None):+ policy = POLICIES[policy_name](capacities, specs, learned_counts)+ for event in trace:+ policy.access(event)+ return policy.stats+++class LayerReplayCache:+ """Memoize exact per-layer replays; all implemented policies are layer-local."""+ def __init__(self, traces, specs, learned_counts=None, base=None):+ self.traces = traces+ self.specs = specs+ self.learned_counts = learned_counts or {}+ self.layer_traces = {+ layer: [[event for event in trace if event.layer == layer] for trace in traces]+ for layer in specs+ }+ self.cache = base.cache if base is not None else {}++ def layer_stats(self, policy_name, layer, capacity, trace_index):+ key = (policy_name, layer, capacity, trace_index)+ if key not in self.cache:+ self.cache[key] = run_policy(+ policy_name,+ {layer: capacity},+ {layer: self.specs[layer]},+ self.layer_traces[layer][trace_index],+ {layer: self.learned_counts.get(layer, {})},+ )+ return self.cache[key]++ def trace_stats(self, policy_name, capacities, specs, trace_index):+ combined = PolicyStats()+ felt_wait_us = 0.0+ for layer, spec in specs.items():+ stats = self.layer_stats(+ policy_name, layer, capacities.get(layer, 0), trace_index)+ combined.add(stats)+ felt_wait_us += stats.misses * spec.felt_miss_us+ combined.felt_wait_us = felt_wait_us+ return combined++ def layer_wait(self, policy_name, layer, capacity, specs):+ misses = sum(+ self.layer_stats(policy_name, layer, capacity, trace_index).misses+ for trace_index in range(len(self.traces))+ )+ return misses * specs[layer].felt_miss_us+++class ScaledReplayCache(LayerReplayCache):+ def __init__(self, base, specs):+ super().__init__(base.traces, specs, base.learned_counts, base=base)+++def evaluate(+ traces,+ capacities,+ specs,+ policies,+ learned_counts=None,+ replay_cache=None,+) -> dict:+ replay_cache = replay_cache or LayerReplayCache(traces, specs, learned_counts)+ result = {}+ for policy_name in policies:+ aggregate = PolicyStats()+ per_trace = []+ for trace_index, trace in enumerate(traces):+ stats = replay_cache.trace_stats(+ policy_name, capacities, specs, trace_index)+ aggregate.add(stats)+ per_trace.append({+ "source": trace[0].source if trace else "",+ **stats.report(),+ })+ result[policy_name] = {"aggregate": aggregate.report(), "traces": per_trace}+ return result+++def capacity_bytes(capacities, specs):+ return sum(capacities.get(layer, 0) * spec.resident_bytes+ for layer, spec in specs.items())+++def scale_specs(specs, felt_scale, target_layer=None):+ if felt_scale <= 0 or not math.isfinite(felt_scale):+ raise ValueError("felt scale must be positive and finite")+ return {+ layer: LayerSpec(+ spec.resident_bytes,+ spec.read_bytes,+ (spec.felt_miss_us * felt_scale+ if target_layer is None or layer == target_layer else spec.felt_miss_us),+ spec.max_experts,+ )+ for layer, spec in specs.items()+ }+++def trace_categories(traces, category_names=None):+ if category_names is not None:+ if len(category_names) != len(traces):+ raise ValueError("--eval-category requires one value per --eval trace")+ return list(category_names)+ categories = []+ for trace in traces:+ source = Path(trace[0].source).stem if trace and trace[0].source else "trace"+ match = re.match(r"^(.*?)(?:[_-]\d+)?$", source)+ categories.append(match.group(1) or source)+ return categories+++def profile_costs(log_paths):+ """Extract per-layer felt costs from PROF logs when layer telemetry exists.++ Current GLM PROF output reports aggregate felt wait, so return the aggregate+ cost-per-miss fallback and leave per-layer calibration to a future layer+ telemetry field.+ """+ waits = []+ loads = []+ for path in log_paths or ():+ text = Path(path).read_text(encoding="utf-8")+ for line in text.splitlines():+ match = re.search(r"([0-9.]+)s read service / ([0-9.]+)s felt wait", line)+ load_match = re.search(r"\| ([0-9]+(?:\.[0-9]+)?) loads/token", line)+ if match:+ waits.append(float(match.group(2)))+ if load_match:+ loads.append(float(load_match.group(1)))+ return {+ "logs": [str(path) for path in log_paths or ()],+ "felt_wait_seconds": sum(waits),+ "loads": sum(loads),+ "felt_us_per_load": (sum(waits) * 1e6 / sum(loads) if loads else None),+ "scope": "aggregate; GLM PROF does not expose layer-level felt wait",+ }+++def decision_gate(result, categories, minimum_gain=0.10, maximum_regression=0.03):+ """Compare every candidate with uniform LRU using equal-weight categories."""+ if not 0 <= minimum_gain < 1 or maximum_regression < 0:+ raise ValueError("invalid decision-gate thresholds")+ baseline_payload = result["results"].get("uniform", {}).get("lru")+ if not baseline_payload:+ raise ValueError("decision gate requires policy lru")+ if len(baseline_payload["traces"]) != len(categories):+ raise ValueError("category count does not match evaluation traces")++ baseline_by_category: dict[str, float] = defaultdict(float)+ category_trace_counts: Counter[str] = Counter(categories)+ for category, stats in zip(categories, baseline_payload["traces"]):+ baseline_by_category[category] += stats["felt_wait_us"]+ rows = []+ for allocation, policies in result["results"].items():+ for policy, payload in policies.items():+ if allocation == "uniform" and policy == "lru":+ continue+ candidate_by_category: dict[str, float] = defaultdict(float)+ for category, stats in zip(categories, payload["traces"]):+ candidate_by_category[category] += stats["felt_wait_us"]+ category_changes = {}+ for category in sorted(baseline_by_category):+ baseline = baseline_by_category[category]+ candidate = candidate_by_category[category]+ change = ((baseline - candidate) / baseline+ if baseline else (0.0 if candidate == 0 else -math.inf))+ category_changes[category] = {+ "baseline_felt_wait_us": baseline,+ "candidate_felt_wait_us": candidate,+ "gain": change,+ "traces": category_trace_counts[category],+ }+ gains = [entry["gain"] for entry in category_changes.values()]+ mean_gain = statistics.fmean(gains)+ worst_gain = min(gains)+ aggregate_baseline = baseline_payload["aggregate"]["felt_wait_us"]+ aggregate_candidate = payload["aggregate"]["felt_wait_us"]+ aggregate_gain = ((aggregate_baseline - aggregate_candidate) / aggregate_baseline+ if aggregate_baseline else 0.0)+ row = {+ "allocation": allocation,+ "policy": policy,+ "aggregate_gain": aggregate_gain,+ "category_mean_gain": mean_gain,+ "worst_category_gain": worst_gain,+ "categories": category_changes,+ }+ row["pass"] = mean_gain >= minimum_gain and worst_gain >= -maximum_regression+ rows.append(row)+ rows.sort(key=lambda row: (+ not row["pass"], -row["worst_category_gain"],+ -row["category_mean_gain"], row["allocation"], row["policy"]))+ return {+ "baseline": {"allocation": "uniform", "policy": "lru"},+ "minimum_category_mean_gain": minimum_gain,+ "maximum_category_regression": maximum_regression,+ "candidates": rows,+ }+++def sensitivity_analysis(+ train,+ evaluation,+ specs,+ budget_bytes,+ policies,+ categories,+ felt_scales,+ minimum_gain=0.10,+ maximum_regression=0.03,+ sensitivity_layers=None,+):+ learned_counts = training_counts(train)+ # Admission and hit/miss behavior is unchanged by a felt-cost scale within+ # one layer. Reuse the exact replay curves across perturbations and only+ # reweight their misses when optimizing dynamic capacity.+ train_cache = LayerReplayCache(train, specs, learned_counts)+ evaluation_cache = LayerReplayCache(evaluation, specs, learned_counts)+ perturbations = [("baseline", None, 1.0)]+ perturbations.extend(+ (f"layer-{layer}-x{felt_scale:g}", layer, felt_scale)+ for layer in (sorted(specs) if sensitivity_layers is None+ else sorted(set(sensitivity_layers)))+ if layer in specs+ for felt_scale in felt_scales+ if felt_scale != 1.0+ )+ scenarios = []+ for name, layer, felt_scale in perturbations:+ scenario_specs = scale_specs(specs, felt_scale, layer)+ scenario_train_cache = ScaledReplayCache(train_cache, scenario_specs)+ scenario_evaluation_cache = ScaledReplayCache(evaluation_cache, scenario_specs)+ result = analyze(+ train, evaluation, scenario_specs, budget_bytes, policies,+ learned_counts, scenario_train_cache, scenario_evaluation_cache)+ scenarios.append({+ "name": name,+ "layer": layer,+ "felt_scale": felt_scale,+ "gate": decision_gate(+ result, categories, minimum_gain, maximum_regression),+ })+ candidates: dict[tuple[str, str], list[dict]] = defaultdict(list)+ for scenario in scenarios:+ for candidate in scenario["gate"]["candidates"]:+ candidates[(candidate["allocation"], candidate["policy"])].append({+ "scenario": scenario["name"],+ "felt_scale": scenario["felt_scale"],+ "pass": candidate["pass"],+ "category_mean_gain": candidate["category_mean_gain"],+ "worst_category_gain": candidate["worst_category_gain"],+ })+ robust = []+ for (allocation, policy), outcomes in candidates.items():+ robust.append({+ "allocation": allocation,+ "policy": policy,+ "pass_all_scales": all(outcome["pass"] for outcome in outcomes),+ "outcomes": outcomes,+ })+ robust.sort(key=lambda row: (not row["pass_all_scales"], row["allocation"], row["policy"]))+ return {"scenarios": scenarios, "candidates": robust}+++def _training_wait(traces, capacities, specs, policy_name, learned_counts):+ return sum(run_policy(policy_name, capacities, specs, trace, learned_counts).felt_wait_us+ for trace in traces)+++def dynamic_capacities(+ traces,+ specs,+ budget_bytes,+ policy_name,+ learned_counts,+ replay_cache=None,+):+ """Greedily allocate slots using exact cold-trace replay for one policy."""+ if budget_bytes < 0:+ raise ValueError("budget must be non-negative")+ capacities = {layer: 0 for layer in specs}+ used = 0+ replay_cache = replay_cache or LayerReplayCache(traces, specs, learned_counts)+ layer_waits = {+ layer: replay_cache.layer_wait(policy_name, layer, 0, specs)+ for layer in specs+ }+ while True:+ best = None+ for layer, spec in specs.items():+ current = capacities[layer]+ # Exact replay is retained for every candidate, but scanning every+ # slot is prohibitively expensive on a 256-expert x 75-layer model.+ # These fixed breakpoints expose the useful capacity regions while+ # keeping the offline sweep bounded and deterministic.+ targets = {1, 2, 4, 8, 16, 32, 48, 64, 96, 128, 160, 192, 256}+ targets.add(spec.max_experts)+ for trace_index in range(len(traces)):+ sequence = replay_cache.layer_traces[layer][trace_index]+ unique = len({expert for event in sequence for expert in event.experts})+ targets.add(min(spec.max_experts, unique))+ targets = {target for target in targets+ if current < target <= spec.max_experts}+ for target in sorted(targets):+ if target <= current:+ continue+ added_bytes = (target - current) * spec.resident_bytes+ if used + added_bytes > budget_bytes:+ break+ wait = replay_cache.layer_wait(policy_name, layer, target, specs)+ benefit = layer_waits[layer] - wait+ candidate = (benefit / added_bytes, benefit, -added_bytes,+ -layer, layer, target, wait, added_bytes)+ if best is None or candidate > best:+ best = candidate+ if best is None or best[1] <= 0:+ break+ (_ratio, _benefit, _neg_bytes, _neg_layer, layer, target,+ layer_wait, added_bytes) = best+ capacities[layer] = target+ layer_waits[layer] = layer_wait+ used += added_bytes+ return capacities+++def analyze(+ train,+ evaluation,+ specs,+ budget_bytes,+ policies,+ learned_counts=None,+ train_cache=None,+ evaluation_cache=None,+):+ learned_counts = learned_counts or training_counts(train)+ train_cache = train_cache or LayerReplayCache(train, specs, learned_counts)+ evaluation_cache = evaluation_cache or LayerReplayCache(+ evaluation, specs, learned_counts)+ uniform = uniform_capacities(specs, budget_bytes)+ output = {+ "budget_bytes": budget_bytes,+ "specs": {str(k): asdict(v) for k, v in sorted(specs.items())},+ "allocations": {},+ "results": {},+ }+ output["allocations"]["uniform"] = {+ "capacities": {str(k): v for k, v in sorted(uniform.items())},+ "resident_bytes": capacity_bytes(uniform, specs),+ }+ output["results"]["uniform"] = evaluate(+ evaluation, uniform, specs, policies, learned_counts, evaluation_cache)+ for policy in policies:+ capacities = dynamic_capacities(+ train, specs, budget_bytes, policy, learned_counts, train_cache)+ name = f"dynamic-{policy}"+ output["allocations"][name] = {+ "capacities": {str(k): v for k, v in sorted(capacities.items())},+ "resident_bytes": capacity_bytes(capacities, specs),+ }+ output["results"][name] = evaluate(+ evaluation, capacities, specs, [policy], learned_counts, evaluation_cache)+ return output+++def print_report(result):+ print(f"expert_budget={result['budget_bytes']} bytes")+ print("allocation policy hit_rate misses bytes_read felt_wait_ms evictions rejected seeded")+ for allocation, policies in result["results"].items():+ caps = result["allocations"][allocation]["capacities"]+ resident = result["allocations"][allocation]["resident_bytes"]+ print(f"capacities[{allocation}]={caps} resident_bytes={resident}")+ for policy, payload in policies.items():+ stats = payload["aggregate"]+ print(f"{allocation:18} {policy:11} {stats['hit_rate']:8.2%} "+ f"{stats['misses']:7d} {stats['bytes_read']:12d} "+ f"{stats['felt_wait_us']/1000:12.3f} "+ f"{stats['evictions']:9d} {stats['rejected_admissions']:8d} "+ f"{stats['seeded_objects']:6d}")+++def print_gate(gate):+ print("decision_gate baseline=uniform/lru "+ f"min_mean_gain={gate['minimum_category_mean_gain']:.1%} "+ f"max_regression={gate['maximum_category_regression']:.1%}")+ print("allocation policy verdict category_mean worst_category aggregate")+ for candidate in gate["candidates"]:+ verdict = "PASS" if candidate["pass"] else "FAIL"+ print(f"{candidate['allocation']:18} {candidate['policy']:11} {verdict:7} "+ f"{candidate['category_mean_gain']:13.2%} "+ f"{candidate['worst_category_gain']:14.2%} "+ f"{candidate['aggregate_gain']:9.2%}")+++def synthetic_trace(kind: str):+ specs = {+ 0: LayerSpec(1_000_000, 1_000_000, 1000.0, 16),+ 1: LayerSpec(1_000_000, 1_000_000, 4000.0, 16),+ 2: LayerSpec(2_000_000, 2_000_000, 2000.0, 16),+ }+ budget = 8_000_000++ def events(layer, sequence):+ return [AccessEvent(layer, (expert,), (expert,), None, kind) for expert in sequence]++ hot0 = [0, 1, 0, 2, 0, 1, 0, 3] * 100+ hot1 = [0, 1, 0, 2, 0, 3, 0, 1, 4, 0, 2, 0] * 100+ burst2 = sum(([0, 1] * 8 + [cold] for cold in range(2, 12)), []) * 20+ train_trace = events(0, hot0) + events(1, hot1) + events(2, burst2)+ if kind == "stationary":+ eval_trace = events(0, hot0[:400]) + events(1, hot1[:600]) + events(2, burst2[:400])+ elif kind == "shifted":+ shifted1 = list(range(16)) * 40+ shifted0 = [7, 8] * 300+ eval_trace = events(0, shifted0) + events(1, shifted1) + events(2, burst2[:300])+ else:+ raise ValueError(kind)+ return [train_trace], [eval_trace], specs, budget+++def synthetic_category_suite():+ """Two-category transfer check: one stationary and one shifted workload."""+ stationary_train, stationary_eval, specs, budget = synthetic_trace("stationary")+ _shifted_train, shifted_eval, _shifted_specs, _shifted_budget = synthetic_trace("shifted")+ return (+ stationary_train,+ [stationary_eval[0], shifted_eval[0]],+ specs,+ budget,+ ["stationary", "shifted"],+ )+++def synthetic_budget_sweep(policies, budgets_mb=(2, 4, 6, 8, 10, 12, 16, 24, 32)):+ train, evaluation, specs, _budget, categories = synthetic_category_suite()+ rows = []+ for budget_mb in budgets_mb:+ budget_bytes = budget_mb * 1_000_000+ result = analyze(train, evaluation, specs, budget_bytes, policies)+ gate = decision_gate(result, categories)+ sensitivity = sensitivity_analysis(+ train, evaluation, specs, budget_bytes, policies, categories,+ [0.5, 1.0, 1.5])+ robust = {+ (row["allocation"], row["policy"]): row["pass_all_scales"]+ for row in sensitivity["candidates"]+ }+ for candidate in gate["candidates"]:+ if candidate["pass"]:+ rows.append({+ "budget_mb": budget_mb,+ "allocation": candidate["allocation"],+ "policy": candidate["policy"],+ "category_mean_gain": candidate["category_mean_gain"],+ "worst_category_gain": candidate["worst_category_gain"],+ "pass_sensitivity": robust[+ (candidate["allocation"], candidate["policy"])],+ })+ return rows+++def command_demo(args):+ for scenario in ("stationary", "shifted"):+ train, evaluation, specs, budget = synthetic_trace(scenario)+ print(f"\n=== {scenario} ===")+ result = analyze(train, evaluation, specs, budget, args.policies)+ print_report(result)+ if "lru" in args.policies:+ print_gate(decision_gate(result, [scenario]))+ if "lru" in args.policies:+ train, evaluation, specs, budget, categories = synthetic_category_suite()+ print("\n=== combined category-held-out gate ===")+ result = analyze(train, evaluation, specs, budget, args.policies)+ print_gate(decision_gate(result, categories))+ sensitivity = sensitivity_analysis(+ train, evaluation, specs, budget, args.policies, categories,+ [0.5, 1.0, 1.5])+ print("sensitivity " + " ".join(+ f"{row['allocation']}/{row['policy']}="+ f"{'PASS' if row['pass_all_scales'] else 'FAIL'}"+ for row in sensitivity["candidates"]))+ sweep = synthetic_budget_sweep(args.policies)+ print("\n=== synthetic budget operating regions ===")+ print("budget_mb allocation policy category_mean worst_category sensitivity")+ for row in sweep:+ print(f"{row['budget_mb']:9d} {row['allocation']:18} {row['policy']:11} "+ f"{row['category_mean_gain']:13.2%} "+ f"{row['worst_category_gain']:14.2%} "+ f"{'PASS' if row['pass_sensitivity'] else 'FAIL'}")+++def _real_inputs(args):+ train_paths = [path.resolve() for path in args.train]+ eval_paths = [path.resolve() for path in args.eval]+ if len(set(train_paths)) != len(train_paths):+ raise ValueError("duplicate path in --train")+ if len(set(eval_paths)) != len(eval_paths):+ raise ValueError("duplicate path in --eval")+ overlap = set(train_paths) & set(eval_paths)+ if overlap:+ raise ValueError(f"train/eval paths overlap: {sorted(map(str, overlap))[0]}")+ try:+ train_identities = [(path.stat().st_dev, path.stat().st_ino) for path in train_paths]+ eval_identities = [(path.stat().st_dev, path.stat().st_ino) for path in eval_paths]+ except OSError:+ train_identities = []+ eval_identities = []+ if len(set(train_identities)) != len(train_identities):+ raise ValueError("duplicate file in --train")+ if len(set(eval_identities)) != len(eval_identities):+ raise ValueError("duplicate file in --eval")+ if set(train_identities) & set(eval_identities):+ raise ValueError("train/eval paths refer to the same file")+ train = read_traces(train_paths)+ evaluation = read_traces(eval_paths)+ specs = load_specs(train + evaluation, args.manifest,+ args.read_gbps, args.felt_fraction)+ return train, evaluation, specs+++def command_run(args):+ train, evaluation, specs = _real_inputs(args)+ if args.max_training_events:+ train = [trace[:args.max_training_events] for trace in train]+ if any(not trace for trace in train):+ raise ValueError("--max-training-events removed every event from a trace")+ budget = int(args.expert_budget_gb * 1e9)+ result = analyze(train, evaluation, specs, budget, args.policies)+ print_report(result)+ categories = trace_categories(evaluation, args.eval_category)+ gate = decision_gate(result, categories, args.minimum_gain, args.maximum_regression)+ print_gate(gate)+ sensitivity = sensitivity_analysis(+ train, evaluation, specs, budget, args.policies, categories,+ args.felt_scales, args.minimum_gain, args.maximum_regression,+ args.sensitivity_layers)+ costs = profile_costs(args.prof_log)+ print("sensitivity " + " ".join(+ f"{row['allocation']}/{row['policy']}="+ f"{'PASS' if row['pass_all_scales'] else 'FAIL'}"+ for row in sensitivity["candidates"]))+ if args.json_out:+ document = {+ "result": result,+ "categories": categories,+ "profile_costs": costs,+ "gate": gate,+ "sensitivity": sensitivity,+ }+ args.json_out.write_text(json.dumps(document, indent=2) + "\n", encoding="utf-8")+++def command_sweep(args):+ train, evaluation, specs = _real_inputs(args)+ if args.max_training_events:+ train = [trace[:args.max_training_events] for trace in train]+ if any(not trace for trace in train):+ raise ValueError("--max-training-events removed every event from a trace")+ categories = trace_categories(evaluation, args.eval_category)+ costs = profile_costs(args.prof_log)+ results = []+ for budget_gb in args.expert_budgets:+ result = analyze(train, evaluation, specs, int(budget_gb * 1e9), args.policies)+ results.append({+ "expert_budget_gb": budget_gb,+ "result": result,+ "profile_costs": costs,+ "gate": decision_gate(+ result, categories, args.minimum_gain, args.maximum_regression),+ "sensitivity": sensitivity_analysis(+ train, evaluation, specs, int(budget_gb * 1e9), args.policies,+ categories, args.felt_scales, args.minimum_gain,+ args.maximum_regression, args.sensitivity_layers),+ })+ for entry in results:+ print(f"\n=== expert budget {entry['expert_budget_gb']:g} GB ===")+ print_report(entry["result"])+ print_gate(entry["gate"])+ print("sensitivity " + " ".join(+ f"{row['allocation']}/{row['policy']}="+ f"{'PASS' if row['pass_all_scales'] else 'FAIL'}"+ for row in entry["sensitivity"]["candidates"]))+ if args.json_out:+ args.json_out.write_text(json.dumps({"sweeps": results}, indent=2) + "\n",+ encoding="utf-8")+++def add_real_trace_arguments(parser):+ parser.add_argument("--train", type=Path, nargs="+", required=True)+ parser.add_argument("--eval", type=Path, nargs="+", required=True)+ parser.add_argument("--manifest", type=Path, required=True)+ parser.add_argument("--read-gbps", type=float, default=3.0)+ parser.add_argument("--felt-fraction", type=float, default=1.0)+ parser.add_argument("--felt-scales", type=float, nargs="+", default=[0.5, 1.0, 1.5])+ parser.add_argument("--eval-category", action="append")+ parser.add_argument("--minimum-gain", type=float, default=0.10)+ parser.add_argument("--maximum-regression", type=float, default=0.03)+ parser.add_argument("--max-training-events", type=int, default=0,+ help="limit events per training trace for a quick smoke run")+ parser.add_argument("--sensitivity-layer", dest="sensitivity_layers",+ type=int, action="append",+ help="limit cost sensitivity to selected layers")+ parser.add_argument("--prof-log", type=Path, action="append", default=[],+ help="GLM PROF=1 log used for aggregate cost calibration")+ parser.add_argument("--policies", nargs="+", choices=tuple(POLICIES),+ default=list(POLICIES))+ parser.add_argument("--json-out", type=Path)+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ sub = parser.add_subparsers(dest="command", required=True)++ demo = sub.add_parser("demo", help="run deterministic stationary/shifted fixtures")+ demo.add_argument("--policies", nargs="+", choices=tuple(POLICIES),+ default=list(POLICIES))+ demo.set_defaults(func=command_demo)++ run = sub.add_parser("run", help="simulate real GLM ROUTE_TRACE files")+ add_real_trace_arguments(run)+ run.add_argument("--expert-budget-gb", type=float, required=True)+ run.set_defaults(func=command_run)++ sweep = sub.add_parser("sweep", help="compare policies over expert RAM budgets")+ add_real_trace_arguments(sweep)+ sweep.add_argument("--expert-budgets", type=float, nargs="+", required=True)+ sweep.set_defaults(func=command_sweep)+ args = parser.parse_args()+ budgets = ([args.expert_budget_gb] if hasattr(args, "expert_budget_gb")+ else getattr(args, "expert_budgets", []))+ if any(value <= 0 or not math.isfinite(value) or value > 1e9 for value in budgets):+ parser.error("expert budget values must be positive, finite, and <= 1e9 GB")+ if hasattr(args, "felt_scales") and any(+ value <= 0 or not math.isfinite(value) for value in args.felt_scales):+ parser.error("--felt-scales values must be positive and finite")+ if hasattr(args, "minimum_gain") and not 0 <= args.minimum_gain < 1:+ parser.error("--minimum-gain must be in [0,1)")+ if hasattr(args, "maximum_regression") and args.maximum_regression < 0:+ parser.error("--maximum-regression must be non-negative")+ if hasattr(args, "max_training_events") and args.max_training_events < 0:+ parser.error("--max-training-events must be non-negative")+ if hasattr(args, "policies") and "lru" not in args.policies:+ parser.error("--policies must include lru as the decision-gate baseline")+ try:+ args.func(args)+ except (OSError, ValueError, json.JSONDecodeError) as error:+ parser.error(str(error))+++if __name__ == "__main__":+ main()diff --git a/docs/experiments/cnre-offline-simulator.md b/docs/experiments/cnre-offline-simulator.mdnew file mode 100644index 00000000..3f65da31--- /dev/null+++ b/docs/experiments/cnre-offline-simulator.md@@ -0,0 +1,281 @@+# CNRE Phase 0: offline residency-policy simulator++This experiment supports [Discussion #884](https://github.com/JustVugg/colibri/discussions/884).+It does not modify an engine or claim a runtime speedup. It replays existing+`ROUTE_TRACE` files through bounded expert caches so weak policies can be+rejected before changing inference code or renting hardware.++## Scope++`c/tools/residency_sim.py` currently models the main GLM CPU-routing path:++- first-seen expert unions for each contiguous `(call, layer)` batch;+- promotion between GLM's 64-expert working-set blocks;+- separate per-layer resident footprint, read bytes, and felt miss cost;+- a fixed expert-residency RAM budget, excluding the rest of process RAM;+- equal slots per layer versus policy-specific, trace-trained layer budgets;+- GLM LRU, synthetic half-capacity pins + LRU, decayed LFU, segmented LRU,+ and decayed-frequency admission.++The half-pinned and alternative admission policies are experiments, not exact+reproductions of current GLM startup pins or live `REPIN`.++The tool does **not** model new prefetch reads, routing substitutions, GPU+kernels, NUMA effects, page-cache interference, lock contention, startup read+cost, or actual tok/s. A predicted reduction in felt wait is a gate for a+runtime A/B, not the result of one.++Current GLM also read-aheads the next 64-expert block while computing the+current block. The simulator preserves block promotion semantics but does not+model this existing overlap. Long-prefill felt-wait estimates are therefore+conservative and must be checked with lower `felt_fraction` sensitivity.++Kimi K3 traces are not currently safe input because that engine emits routes+without advancing the trace call ID. Inkling and OLMoE initialize routing+telemetry but do not currently emit route records. Their cache traversal also+differs from GLM, so each needs an explicit engine profile before inclusion.++## Trace contract++The input is the engine's current `ROUTE_TRACE` stream:++```text+<call> <row> <layer> <expert>:<gate> ...+```++Rows sharing one contiguous `(call, layer)` group are one runtime batch. The+simulator unions their expert IDs in first-seen order for demand I/O, while+retaining per-row selection multiplicity to train startup residents like+`.coli_usage` counts. It rejects malformed fields, missing/reordered rows,+non-advancing call IDs, and repeated noncontiguous call groups rather than+silently merging a broken trace.++Use a clean environment and deterministic, placement-neutral GLM trace+collection. In particular, unset `EXPERT_BUDGET` and `ABLATE_*`; the former is+applied after tracing and would make the trace differ from executed demand.+Record `COLI_PREFILL_CHUNK`, because it changes batch boundaries.++For a large model, place the model and traces on the instance's local scratch+volume rather than the root disk. On DigitalOcean's H200 image this is commonly+`/dev/vdc1`, mounted at `/mnt/scratch`; verify with `lsblk` and `findmnt` before+using it. The scratch volume is ephemeral and must not be treated as durable+storage.++```sh+env -i PATH="$PATH" HOME="$HOME" \+ DRAFT=0 MTP=0 TOPP=0 TOPK=0 CACHE_ROUTE=0 \+ PILOT=0 PILOT_REAL=0 REPIN=0 TEMP=0 \+ ROUTE_TRACE=trace.txt \+ SNAP=/path/to/model PROMPT="..." NGEN=128 ./colibri 64 4 8+```++Training traces and held-out evaluation traces must be separate files. The CLI+rejects identical resolved paths. Use one file per cold process run; policy+state is intentionally reset between files.++## Manifest and cost calibration++A manifest with fixed model geometry is required. Inferring `max_experts` from+held-out IDs would leak evaluation data and understate dimensions when a valid+expert did not happen to route.++```json+{+ "layers": {+ "3": {+ "resident_bytes": 18915328,+ "read_bytes": 75661312,+ "felt_miss_us": 2400.0,+ "max_experts": 256+ }+ }+}+```++Manifest rows that are absent from all supplied traces are ignored. This keeps+disabled MTP rows from consuming only the uniform baseline's budget.++`resident_bytes` consumes the expert cache budget. `read_bytes` counts demand+I/O and may differ when a source tensor is transformed into a smaller resident+representation. If `felt_miss_us` is omitted, it defaults to:++```text+read_bytes / (read_gbps * 1e9) * 1e6 * felt_fraction+```++`felt_fraction` is the fraction of read service not hidden by concurrent+compute. Caching a 20 ms read that is fully overlapped does not remove 20 ms+from token latency.++## Commands++Run deterministic fixtures:++```sh+cd c+python3 tools/residency_sim.py demo+python3 -m unittest tests.test_residency_sim+```++Evaluate a fixed expert-cache budget:++```sh+python3 tools/residency_sim.py run \+ --train traces/chat_train.trace traces/code_train.trace \+ --eval traces/chat_heldout_1.trace traces/code_heldout_1.trace \+ --eval-category chat --eval-category code \+ --manifest glm-residency.json \+ --expert-budget-gb 32 \+ --read-gbps 3.0 \+ --felt-fraction 0.7 \+ --felt-scales 0.5 1.0 1.5 \+ --sensitivity-layer 3 --sensitivity-layer 30 --sensitivity-layer 60 \+ --prof-log /mnt/scratch/logs/trace/coding_eval.out \+ --json-out cnre-32gb.json+```++Sweep expert-cache capacities:++```sh+python3 tools/residency_sim.py sweep \+ --train traces/chat_train.trace traces/code_train.trace \+ --eval traces/chat_heldout_1.trace traces/code_heldout_1.trace \+ --eval-category chat --eval-category code \+ --manifest glm-residency.json \+ --expert-budgets 8 16 24 32 48 64 \+ --read-gbps 3.0 \+ --felt-fraction 0.7 \+ --json-out cnre-sweep.json+```++JSON preserves model specs, every allocation, per-trace source paths and+metrics, and each complete sweep result. This is enough to compute category+regressions outside the simulator without parsing human output.++`--policies` must include `lru`, because every decision is measured against the+uniform-LRU baseline. Both `run` and every point in `sweep` report nominal and+layer-cost-sensitivity verdicts.++`--prof-log` stores the aggregate `PROF=1` felt-wait calibration in the JSON.+Current GLM logs expose aggregate, not per-layer, felt wait; this is provenance+and a global calibration check, not a claim of layer-specific measurement.++`run` also applies the Phase-0 decision gate directly. Categories receive equal+weight regardless of their number or length of traces. A candidate passes only+when its category-mean felt-wait gain is at least 10% and its worst category is+not more than 3% slower than uniform LRU. If `--eval-category` is omitted, the+tool derives categories by stripping a final numeric replicate from file names,+for example `chat_2.trace` -> `chat`.++Sensitivity reruns the complete training allocation and held-out evaluation+after scaling each layer's felt miss cost independently. `--felt-scales 0.5 1+1.5` therefore tests the baseline plus low/high cost perturbations for every+active layer. Uniformly scaling every layer would leave policy rankings and+percentage gains unchanged and is not a useful robustness test.++For a large model, use a representative cost sample first, for example+`--sensitivity-layer 3 --sensitivity-layer 30 --sensitivity-layer 60`. The+default all-layer sensitivity is intentionally thorough but can be expensive+because it recomputes policy-specific capacity allocation for every perturbation.++## Deterministic fixtures++The built-in stationary workload is intentionally cacheable and demonstrates+scan resistance. The shifted held-out workload deliberately changes its hot+set and demonstrates that training-seeded residents and dynamic layer budgets+can overfit. Exact fixture numbers are printed by `demo` and asserted only where+they encode an invariant; they are not evidence about real Colibri workloads.++`demo` then evaluates both fixtures as equal-weight categories in one decision+gate. This is deliberately stricter than averaging all accesses: a large+stationary win cannot hide a shifted-category regression beyond 3%. In the+current fixture, uniform frequency admission fails that guard (`-4.76%` in the+shifted category). Dynamic frequency admission narrowly passes the nominal+gate (`+4.27%` worst category) but fails the layer-by-layer felt-cost+sensitivity check at that 8 MB budget.++The final demo table sweeps synthetic expert budgets from 2 to 32 MB. It shows+that robustness is an operating region rather than a policy-wide property. For+example, dynamic frequency admission is robust in the deliberately constrained+2-6 MB fixture, fragile at 8-12 MB, and robust again at larger toy budgets. These+transitions are artifacts of the fixture, but they demonstrate why real traces+must be swept across realistic RAM budgets rather than evaluated at one point.++## Phase-0 decision gate++Advance to runtime telemetry or an opt-in cache policy only if category-held-out+real traces show:++```text+at least 10% lower predicted felt wait+no principal held-out category worse by more than 3%+bounded churn and exact expert-cache byte-budget compliance+the result survives conservative miss-cost sensitivity+```++The expert-cache budget does not establish whole-process RAM compliance. A+runtime implementation must additionally account for dense weights, KV state,+working-set slabs, scratch, page cache, and OS reserve. A negative result is a+completed experiment and should be published.++## Data still needed++The repository does not currently contain complete GLM routing traces. The+first useful campaign needs:++1. GLM-5.2 training and held-out traces from coding, chat, multilingual,+ reasoning, and long-context prompts.+2. Matching `PROF=1` logs or a per-layer felt-cost manifest.+3. Exact commit, model/container identity, DRAFT/sampling settings, cache state,+ prefill chunk, and frozen `.coli_usage` snapshot.+4. Separate instrumentation fixes and engine profiles before Kimi K3, Inkling,+ or OLMoE data is interpreted by this simulator.++No server is required to run the simulator. A model-capable machine is needed+only to collect missing traces and later validate a policy end to end.++## H200 pilot result++The first real-trace campaign ran on a DigitalOcean H200 instance using the+gs64 GLM container at revision+`95bb5f03e3e0ca16b4c711394d0461fa86a3cfcb`. The model occupied 429.3 GB on the+5 TB local scratch volume. The machine had 24 physical CPU cores, 247 GB RAM,+and 143.8 GiB H200 VRAM.++The collection produced eight training traces and five held-out traces across+coding, chat, reasoning, multilingual, and long-context prompts. These are+small pilot traces, not a production benchmark corpus.++At expert-cache budgets of 120, 160, and 200 GB, the offline simulator reported:++```text+budget candidate mean held-out gain worst category+120 GB uniform frequency +39.91% +34.84%+120 GB dynamic frequency +40.45% +35.41%+160 GB uniform frequency +53.35% +49.92%+160 GB dynamic frequency +54.50% +50.66%+200 GB uniform frequency +65.31% +62.64%+200 GB dynamic frequency +66.29% +63.07%+```++All frequency candidates passed the nominal category gate and the sampled+layer-cost sensitivity at layers 3, 30, and 60. LRU dynamic allocation did not+meet the 10% gate, with approximately 0.0-0.4% gain.++The runtime pilot is the necessary qualification. With `RAM_GB=120`, `PIPE=0`+measured 1.66 tok/s at 57.2% hit; with `RAM_GB=200`, it measured 1.31 tok/s at+73.5% hit. Enabling the existing `PIPE=1` overlap reached 2.00 tok/s at the+same 57.2% hit. This means hit rate alone is not a performance result: I/O+overlap, disk service, and compute contention materially change tok/s.++Across the five held-out `PROF=1` logs, the aggregate runtime instrumentation+reported 19.6 seconds of felt wait over 2,625 loads, or approximately 7.47 ms+per load. This is a whole-run calibration signal, not a per-layer cost: the+current GLM profile does not attribute felt wait to individual layers.++No runtime frequency-admission policy was implemented in this pilot. The+current GLM cache admits every demand miss in `moe()` and promotes the bounded+tail of each 64-expert block. An unrecognized `COLI_CACHE_ADMISSION` variable+therefore has no effect. This is intentional: the real-trace evidence is strong+enough to justify a narrowly scoped opt-in prototype, but not enough to claim a+runtime win or to make a misleading A/B number part of the RFC.