Skip to content

Personal AI, On Personal Devices

Python

fix(telemetry): isolate events and unsubscribe stores (#887)

Elliot Sluskyfccb2acc2e19
diff --git a/src/openjarvis/core/events.py b/src/openjarvis/core/events.py
index d248c03b..231a7be4 100644
--- a/src/openjarvis/core/events.py
+++ b/src/openjarvis/core/events.py
@@ -7,12 +7,15 @@ react without direct coupling.
from __future__ import annotations
+import logging
import threading
import time
from dataclasses import dataclass, field
from enum import Enum
from typing import Any, Callable, Dict, List, Optional # noqa: I001
+logger = logging.getLogger(__name__)
+
# ---------------------------------------------------------------------------
# Event taxonomy
# ---------------------------------------------------------------------------
@@ -147,7 +150,16 @@ class EventBus:
listeners = list(self._subscribers.get(event_type, []))
for callback in listeners:
- callback(event)
+ try:
+ callback(event)
+ except Exception:
+ # Subscribers are observers: one broken observer must neither
+ # interrupt the publisher nor starve later subscribers.
+ logger.exception(
+ "Event subscriber %r failed while handling %s",
+ callback,
+ event_type.value,
+ )
return event
diff --git a/src/openjarvis/telemetry/store.py b/src/openjarvis/telemetry/store.py
index b93ea873..617e78f8 100644
--- a/src/openjarvis/telemetry/store.py
+++ b/src/openjarvis/telemetry/store.py
@@ -207,6 +207,7 @@ class TelemetryStore:
self._telemetry_batch: list[tuple[Any, ...]] = []
self._mining_batch: list[tuple[Any, ...]] = []
self._closed = False
+ self._event_bus: EventBus | None = None
# Background flusher: without it, a partial batch written just before
# traffic stops would stay invisible to other connections until the
@@ -293,6 +294,8 @@ class TelemetryStore:
json.dumps(rec.metadata),
)
with self._lock:
+ if self._closed:
+ raise RuntimeError("TelemetryStore is closed")
self._telemetry_batch.append(row)
self._maybe_flush_unlocked()
@@ -316,6 +319,8 @@ class TelemetryStore:
stats.fees_owed,
)
with self._lock:
+ if self._closed:
+ raise RuntimeError("TelemetryStore is closed")
self._mining_batch.append(row)
self._maybe_flush_unlocked()
@@ -369,7 +374,15 @@ class TelemetryStore:
def subscribe_to_bus(self, bus: EventBus) -> None:
"""Subscribe to ``TELEMETRY_RECORD`` events on *bus*."""
- bus.subscribe(EventType.TELEMETRY_RECORD, self._on_event)
+ with self._lock:
+ if self._closed:
+ raise RuntimeError("TelemetryStore is closed")
+ if self._event_bus is bus:
+ return
+ if self._event_bus is not None:
+ self._event_bus.unsubscribe(EventType.TELEMETRY_RECORD, self._on_event)
+ bus.subscribe(EventType.TELEMETRY_RECORD, self._on_event)
+ self._event_bus = bus
def _on_event(self, event: Event) -> None:
rec = event.data.get("record")
@@ -388,6 +401,12 @@ class TelemetryStore:
with self._lock:
if self._closed:
return
+ if self._event_bus is not None:
+ self._event_bus.unsubscribe(
+ EventType.TELEMETRY_RECORD,
+ self._on_event,
+ )
+ self._event_bus = None
self._flush_unlocked()
self._conn.close()
self._closed = True
diff --git a/tests/core/test_events.py b/tests/core/test_events.py
index 471b6270..5955ca5b 100644
--- a/tests/core/test_events.py
+++ b/tests/core/test_events.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import logging
import threading
from openjarvis.core.events import (
@@ -32,6 +33,38 @@ class TestEventBus:
assert len(a) == 1
assert len(b) == 1
+ def test_failing_subscriber_does_not_starve_later_subscribers(self, caplog) -> None:
+ bus = EventBus()
+ received: list[Event] = []
+
+ def fail(_event: Event) -> None:
+ raise ValueError("boom")
+
+ bus.subscribe(EventType.INFERENCE_END, fail)
+ bus.subscribe(EventType.INFERENCE_END, received.append)
+
+ with caplog.at_level(logging.ERROR, logger="openjarvis.core.events"):
+ event = bus.publish(EventType.INFERENCE_END, {"model": "test"})
+
+ assert received == [event]
+ assert "boom" in caplog.text
+ assert "inference_end" in caplog.text
+
+ def test_publish_does_not_swallow_process_control_exceptions(self) -> None:
+ bus = EventBus()
+
+ def interrupt(_event: Event) -> None:
+ raise KeyboardInterrupt
+
+ bus.subscribe(EventType.INFERENCE_END, interrupt)
+
+ try:
+ bus.publish(EventType.INFERENCE_END)
+ except KeyboardInterrupt:
+ pass
+ else: # pragma: no cover - regression assertion
+ raise AssertionError("KeyboardInterrupt should propagate")
+
def test_unsubscribe(self) -> None:
bus = EventBus()
received: list[Event] = []
diff --git a/tests/telemetry/test_store.py b/tests/telemetry/test_store.py
index 7f2270dc..90c0a82b 100644
--- a/tests/telemetry/test_store.py
+++ b/tests/telemetry/test_store.py
@@ -82,6 +82,64 @@ class TestTelemetryStore:
assert rows[0][2] == "test-model"
store.close()
+ def test_bus_subscription_is_idempotent(self, tmp_path: Path) -> None:
+ store = TelemetryStore(tmp_path / "test.db")
+ bus = EventBus()
+ store.subscribe_to_bus(bus)
+ store.subscribe_to_bus(bus)
+
+ rec = TelemetryRecord(timestamp=time.time(), model_id="once", engine="test")
+ bus.publish(EventType.TELEMETRY_RECORD, {"record": rec})
+
+ assert len(store._fetchall()) == 1
+ store.close()
+
+ def test_resubscribe_moves_store_to_new_bus(self, tmp_path: Path) -> None:
+ store = TelemetryStore(tmp_path / "test.db")
+ old_bus = EventBus()
+ new_bus = EventBus()
+ store.subscribe_to_bus(old_bus)
+ store.subscribe_to_bus(new_bus)
+
+ rec = TelemetryRecord(timestamp=time.time(), model_id="new", engine="test")
+ old_bus.publish(EventType.TELEMETRY_RECORD, {"record": rec})
+ new_bus.publish(EventType.TELEMETRY_RECORD, {"record": rec})
+
+ assert len(store._fetchall()) == 1
+ store.close()
+
+ def test_close_unsubscribes_from_bus_and_is_idempotent(
+ self, tmp_path: Path
+ ) -> None:
+ store = TelemetryStore(tmp_path / "test.db")
+ bus = EventBus()
+ store.subscribe_to_bus(bus)
+
+ store.close()
+ store.close()
+ bus.publish(
+ EventType.TELEMETRY_RECORD,
+ {
+ "record": TelemetryRecord(
+ timestamp=time.time(), model_id="late", engine="test"
+ )
+ },
+ )
+
+ assert store._event_bus is None
+ assert bus._subscribers[EventType.TELEMETRY_RECORD] == []
+
+ def test_closed_store_rejects_new_subscription(self, tmp_path: Path) -> None:
+ store = TelemetryStore(tmp_path / "test.db")
+ store.close()
+
+ try:
+ store.subscribe_to_bus(EventBus())
+ except RuntimeError as exc:
+ assert str(exc) == "TelemetryStore is closed"
+ else: # pragma: no cover - regression assertion
+ raise AssertionError("closed store accepted a new subscription")
+
def test_close_and_reopen(self, tmp_path: Path) -> None:
db_path = tmp_path / "test.db"
store = TelemetryStore(db_path)