Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

cluster: measure worker lease expiry with a monotonic clock

Rudy Celekli44f3176ecd8a
diff --git a/c/cluster.py b/c/cluster.py
index 05fa3819..efa2964e 100644
--- a/c/cluster.py
+++ b/c/cluster.py
@@ -20,6 +20,7 @@ class ClusterRegistry:
def __init__(self, stale_after=30.0):
self.stale_after = float(stale_after)
self._nodes = {}
+ self._last_seen = {}
self._lock = threading.Lock()
def register(self, node):
@@ -43,6 +44,7 @@ class ClusterRegistry:
last_seen=time.time())
with self._lock:
self._nodes[node_id] = record
+ self._last_seen[node_id] = time.monotonic()
return record
def heartbeat(self, node_id):
@@ -51,13 +53,14 @@ class ClusterRegistry:
if node is None:
raise KeyError(node_id)
node["last_seen"] = time.time()
+ self._last_seen[str(node_id)] = time.monotonic()
return dict(node)
def snapshot(self):
- now = time.time()
+ now = time.monotonic()
with self._lock:
- nodes = [dict(node) for node in self._nodes.values()
- if now - node["last_seen"] <= self.stale_after]
+ nodes = [dict(node) for node_id, node in self._nodes.items()
+ if now - self._last_seen[node_id] <= self.stale_after]
nodes.sort(key=lambda node: (node["role"], node["node_id"]))
return {"protocol_version": PROTOCOL_VERSION, "nodes": nodes}
diff --git a/c/tests/test_cluster_clock.py b/c/tests/test_cluster_clock.py
new file mode 100644
index 00000000..215b5961
--- /dev/null
+++ b/c/tests/test_cluster_clock.py
@@ -0,0 +1,45 @@
+"""Worker leases measure elapsed time while public timestamps stay wall time."""
+import unittest
+from unittest.mock import patch
+from types import SimpleNamespace
+from cluster import ClusterRegistry
+
+
+class ClusterLeaseClockTest(unittest.TestCase):
+ def test_forward_wall_clock_change_does_not_expire_live_worker(self):
+ registry = ClusterRegistry(stale_after=30)
+ with patch('cluster.time', SimpleNamespace(time=lambda: 1000, monotonic=lambda: 10)):
+ record = registry.register({'node_id': 'worker', 'host': 'localhost', 'port': 9100, 'role': 'expert'})
+ self.assertEqual(record['last_seen'], 1000)
+ with patch('cluster.time', SimpleNamespace(time=lambda: 4600, monotonic=lambda: 11)):
+ self.assertEqual(registry.expert_endpoints(), ['localhost:9100'])
+
+ def test_backward_wall_clock_change_does_not_keep_dead_worker(self):
+ registry = ClusterRegistry(stale_after=30)
+ with patch('cluster.time', SimpleNamespace(time=lambda: 1000, monotonic=lambda: 10)):
+ registry.register({'node_id': 'worker', 'host': 'localhost', 'port': 9100, 'role': 'expert'})
+ with patch('cluster.time', SimpleNamespace(time=lambda: 100, monotonic=lambda: 41)):
+ self.assertEqual(registry.expert_endpoints(), [])
+
+ def test_lease_lookup_uses_registry_key_not_returned_record_id(self):
+ registry = ClusterRegistry(stale_after=30)
+ record = registry.register({'node_id': 'worker', 'host': 'localhost', 'port': 9100, 'role': 'expert'})
+ # register has historically returned the record itself. Its public
+ # fields must not become keys into the private elapsed-time map.
+ record['node_id'] = 'display-id'
+ self.assertEqual(registry.expert_endpoints(), ['localhost:9100'])
+
+ def test_heartbeat_renews_lease_and_keeps_public_wall_timestamp(self):
+ registry = ClusterRegistry(stale_after=30)
+ with patch('cluster.time', SimpleNamespace(time=lambda: 1000, monotonic=lambda: 10)):
+ registry.register({'node_id': 1, 'host': 'localhost', 'port': 9100, 'role': 'expert'})
+ with patch('cluster.time', SimpleNamespace(time=lambda: 500, monotonic=lambda: 39)):
+ self.assertEqual(registry.heartbeat(1)['last_seen'], 500)
+ with patch('cluster.time', SimpleNamespace(time=lambda: 800, monotonic=lambda: 68)):
+ self.assertEqual(registry.expert_endpoints(), ['localhost:9100'])
+ with patch('cluster.time', SimpleNamespace(time=lambda: 800, monotonic=lambda: 70)):
+ self.assertEqual(registry.expert_endpoints(), [])
+
+
+if __name__ == '__main__':
+ unittest.main()