Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
cluster: measure worker lease expiry with a monotonic clock
Rudy Celekli44f3176ecd8a
diff --git a/c/cluster.py b/c/cluster.pyindex 05fa3819..efa2964e 100644--- a/c/cluster.py+++ b/c/cluster.py@@ -20,6 +20,7 @@ class ClusterRegistry:def __init__(self, stale_after=30.0):self.stale_after = float(stale_after)self._nodes = {}+ self._last_seen = {}self._lock = threading.Lock()def register(self, node):@@ -43,6 +44,7 @@ class ClusterRegistry:last_seen=time.time())with self._lock:self._nodes[node_id] = record+ self._last_seen[node_id] = time.monotonic()return recorddef heartbeat(self, node_id):@@ -51,13 +53,14 @@ class ClusterRegistry:if node is None:raise KeyError(node_id)node["last_seen"] = time.time()+ self._last_seen[str(node_id)] = time.monotonic()return dict(node)def snapshot(self):- now = time.time()+ now = time.monotonic()with self._lock:- nodes = [dict(node) for node in self._nodes.values()- if now - node["last_seen"] <= self.stale_after]+ nodes = [dict(node) for node_id, node in self._nodes.items()+ if now - self._last_seen[node_id] <= self.stale_after]nodes.sort(key=lambda node: (node["role"], node["node_id"]))return {"protocol_version": PROTOCOL_VERSION, "nodes": nodes}diff --git a/c/tests/test_cluster_clock.py b/c/tests/test_cluster_clock.pynew file mode 100644index 00000000..215b5961--- /dev/null+++ b/c/tests/test_cluster_clock.py@@ -0,0 +1,45 @@+"""Worker leases measure elapsed time while public timestamps stay wall time."""+import unittest+from unittest.mock import patch+from types import SimpleNamespace+from cluster import ClusterRegistry+++class ClusterLeaseClockTest(unittest.TestCase):+ def test_forward_wall_clock_change_does_not_expire_live_worker(self):+ registry = ClusterRegistry(stale_after=30)+ with patch('cluster.time', SimpleNamespace(time=lambda: 1000, monotonic=lambda: 10)):+ record = registry.register({'node_id': 'worker', 'host': 'localhost', 'port': 9100, 'role': 'expert'})+ self.assertEqual(record['last_seen'], 1000)+ with patch('cluster.time', SimpleNamespace(time=lambda: 4600, monotonic=lambda: 11)):+ self.assertEqual(registry.expert_endpoints(), ['localhost:9100'])++ def test_backward_wall_clock_change_does_not_keep_dead_worker(self):+ registry = ClusterRegistry(stale_after=30)+ with patch('cluster.time', SimpleNamespace(time=lambda: 1000, monotonic=lambda: 10)):+ registry.register({'node_id': 'worker', 'host': 'localhost', 'port': 9100, 'role': 'expert'})+ with patch('cluster.time', SimpleNamespace(time=lambda: 100, monotonic=lambda: 41)):+ self.assertEqual(registry.expert_endpoints(), [])++ def test_lease_lookup_uses_registry_key_not_returned_record_id(self):+ registry = ClusterRegistry(stale_after=30)+ record = registry.register({'node_id': 'worker', 'host': 'localhost', 'port': 9100, 'role': 'expert'})+ # register has historically returned the record itself. Its public+ # fields must not become keys into the private elapsed-time map.+ record['node_id'] = 'display-id'+ self.assertEqual(registry.expert_endpoints(), ['localhost:9100'])++ def test_heartbeat_renews_lease_and_keeps_public_wall_timestamp(self):+ registry = ClusterRegistry(stale_after=30)+ with patch('cluster.time', SimpleNamespace(time=lambda: 1000, monotonic=lambda: 10)):+ registry.register({'node_id': 1, 'host': 'localhost', 'port': 9100, 'role': 'expert'})+ with patch('cluster.time', SimpleNamespace(time=lambda: 500, monotonic=lambda: 39)):+ self.assertEqual(registry.heartbeat(1)['last_seen'], 500)+ with patch('cluster.time', SimpleNamespace(time=lambda: 800, monotonic=lambda: 68)):+ self.assertEqual(registry.expert_endpoints(), ['localhost:9100'])+ with patch('cluster.time', SimpleNamespace(time=lambda: 800, monotonic=lambda: 70)):+ self.assertEqual(registry.expert_endpoints(), [])+++if __name__ == '__main__':+ unittest.main()