Personal AI, On Personal Devices
Python
fix(telemetry): record successful stream_full inferences (#1020)
Johnny C Wattccce49692ca0
diff --git a/src/openjarvis/telemetry/instrumented_engine.py b/src/openjarvis/telemetry/instrumented_engine.pyindex 129bdb3a..55cf5086 100644--- a/src/openjarvis/telemetry/instrumented_engine.py+++ b/src/openjarvis/telemetry/instrumented_engine.py@@ -315,79 +315,55 @@ class InstrumentedEngine(InferenceEngine):return result- async def stream(+ def _publish_stream_telemetry(self,- messages: Sequence[Message],*,- model: str = "",- temperature: float = 0.7,- max_tokens: int = 1024,- **kwargs: Any,- ) -> Any:- """Stream with per-token timing and full telemetry recording."""- self._bus.publish(- EventType.INFERENCE_START,- {- "model": model,- "message_count": len(messages),- },- )-- t0 = time.time()- token_timestamps: list[float] = []- token_count = 0-- energy_sample: Optional[Any] = None- gpu_sample: Optional[GpuSample] = None-- if self._energy_monitor is not None:- with self._energy_monitor.sample() as energy_sample:- async for token in self._inner.stream(- messages,- model=model,- temperature=temperature,- max_tokens=max_tokens,- **kwargs,- ):- token_timestamps.append(time.time())- token_count += 1- yield token- elif self._gpu_monitor is not None:- with self._gpu_monitor.sample() as gpu_sample:- async for token in self._inner.stream(- messages,- model=model,- temperature=temperature,- max_tokens=max_tokens,- **kwargs,- ):- token_timestamps.append(time.time())- token_count += 1- yield token+ model: str,+ t0: float,+ token_timestamps: list[float],+ fallback_token_count: int,+ energy_sample: Optional[Any],+ gpu_sample: Optional[GpuSample],+ usage: Optional[Dict[str, Any]] = None,+ ) -> None:+ """Publish telemetry after a streaming call completes successfully."""+ latency = time.time() - t0+ usage_data = usage or {}++ if usage is None:+ prompt_tokens = 0+ prompt_tokens_evaluated = 0+ completion_tokens = fallback_token_count+ # Preserve stream()'s existing record shape when provider usage is+ # unavailable. Rich streams with usage populate the exact total.+ total_tokens = 0else:- async for token in self._inner.stream(- messages,- model=model,- temperature=temperature,- max_tokens=max_tokens,- **kwargs,- ):- token_timestamps.append(time.time())- token_count += 1- yield token+ prompt_tokens = usage_data.get("prompt_tokens", 0) or 0+ prompt_tokens_evaluated = (+ usage_data.get("prompt_tokens_evaluated") or prompt_tokens+ )+ reported_completion = usage_data.get("completion_tokens")+ completion_tokens = (+ fallback_token_count+ if reported_completion is None+ else reported_completion+ )+ reported_total = usage_data.get("total_tokens")+ total_tokens = (+ prompt_tokens + completion_tokens+ if reported_total is None+ else reported_total+ )- latency = time.time() - t0ttft = token_timestamps[0] - t0 if token_timestamps else 0.0- throughput = token_count / latency if latency > 0 else 0.0+ throughput = completion_tokens / latency if latency > 0 else 0.0- # Compute ITL from consecutive timestampsitl_values_ms = [(token_timestamps[i] - token_timestamps[i - 1]) * 1000for i in range(1, len(token_timestamps))]itl_stats = _compute_itl_stats(itl_values_ms)- # Energy / GPU metrics from sampleenergy_joules = 0.0power_watts = 0.0gpu_utilization_pct = 0.0@@ -424,20 +400,16 @@ class InstrumentedEngine(InferenceEngine):gpu_temperature_c = gpu_sample.mean_temperature_cenergy_method = "polling"energy_vendor = "nvidia"- # The legacy monitor reads one GPU rail and nothing else.gpu_energy_joules = gpu_sample.energy_joulessoc_energy_joules = gpu_sample.energy_joulesenergy_basis = BASIS_GPUprefill_latency = ttft if ttft > 0 else 0.0-- # Derived metricsenergy_per_output_token = (- energy_joules / token_count if token_count > 0 else 0.0+ energy_joules / completion_tokens if completion_tokens > 0 else 0.0)throughput_per_watt = throughput / power_watts if power_watts > 0 else 0.0- # Phase energy splitdecode_latency = latency - prefill_latency if prefill_latency > 0 else 0.0prefill_energy = 0.0decode_energy = 0.0@@ -446,19 +418,20 @@ class InstrumentedEngine(InferenceEngine):prefill_energy = energy_joules * prefill_fracdecode_energy = energy_joules * (1.0 - prefill_frac)- # Per-inference efficiencytokens_per_joule = (- token_count / energy_joules- if energy_joules > 0 and token_count > 0+ completion_tokens / energy_joules+ if energy_joules > 0 and completion_tokens > 0else 0.0)engine_id = getattr(self._inner, "engine_id", "unknown")-record = TelemetryRecord(timestamp=t0,model_id=model,- completion_tokens=token_count,+ prompt_tokens=prompt_tokens,+ prompt_tokens_evaluated=prompt_tokens_evaluated,+ completion_tokens=completion_tokens,+ total_tokens=total_tokens,latency_seconds=latency,ttft=ttft,throughput_tok_per_sec=throughput,@@ -490,7 +463,6 @@ class InstrumentedEngine(InferenceEngine):soc_energy_joules=soc_energy_joules,energy_basis=energy_basis,tokens_per_joule=tokens_per_joule,- # Stamp the methodology version on streaming records too.token_counting_version=TOKEN_COUNTING_VERSION,)@@ -499,7 +471,7 @@ class InstrumentedEngine(InferenceEngine):"latency": latency,"ttft": ttft,"throughput_tok_per_sec": throughput,- "completion_tokens": token_count,+ "completion_tokens": completion_tokens,"is_streaming": True,"mean_itl_ms": itl_stats["mean"],"median_itl_ms": itl_stats["median"],@@ -509,10 +481,82 @@ class InstrumentedEngine(InferenceEngine):"energy_method": energy_method,"energy_vendor": energy_vendor,}+ if usage is not None:+ event_data["usage"] = usage_dataself._bus.publish(EventType.INFERENCE_END, event_data)self._bus.publish(EventType.TELEMETRY_RECORD, {"record": record})+ async def stream(+ self,+ messages: Sequence[Message],+ *,+ model: str = "",+ temperature: float = 0.7,+ max_tokens: int = 1024,+ **kwargs: Any,+ ) -> Any:+ """Stream with per-token timing and full telemetry recording."""+ self._bus.publish(+ EventType.INFERENCE_START,+ {+ "model": model,+ "message_count": len(messages),+ },+ )++ t0 = time.time()+ token_timestamps: list[float] = []+ token_count = 0++ energy_sample: Optional[Any] = None+ gpu_sample: Optional[GpuSample] = None++ if self._energy_monitor is not None:+ with self._energy_monitor.sample() as energy_sample:+ async for token in self._inner.stream(+ messages,+ model=model,+ temperature=temperature,+ max_tokens=max_tokens,+ **kwargs,+ ):+ token_timestamps.append(time.time())+ token_count += 1+ yield token+ elif self._gpu_monitor is not None:+ with self._gpu_monitor.sample() as gpu_sample:+ async for token in self._inner.stream(+ messages,+ model=model,+ temperature=temperature,+ max_tokens=max_tokens,+ **kwargs,+ ):+ token_timestamps.append(time.time())+ token_count += 1+ yield token+ else:+ async for token in self._inner.stream(+ messages,+ model=model,+ temperature=temperature,+ max_tokens=max_tokens,+ **kwargs,+ ):+ token_timestamps.append(time.time())+ token_count += 1+ yield token++ self._publish_stream_telemetry(+ model=model,+ t0=t0,+ token_timestamps=token_timestamps,+ fallback_token_count=token_count,+ energy_sample=energy_sample,+ gpu_sample=gpu_sample,+ )+async def stream_full(self,messages: Sequence[Message],@@ -522,15 +566,80 @@ class InstrumentedEngine(InferenceEngine):max_tokens: int = 1024,**kwargs: Any,) -> AsyncIterator["StreamChunk"]:- """Delegate to inner engine's stream_full for tool-call support."""- async for chunk in self._inner.stream_full(- messages,+ """Stream rich chunks with timing, usage, and energy telemetry.++ Telemetry is published only after the inner stream completes+ successfully.+ """+ self._bus.publish(+ EventType.INFERENCE_START,+ {+ "model": model,+ "message_count": len(messages),+ },+ )++ t0 = time.time()+ token_timestamps: list[float] = []+ content_chunk_count = 0+ usage: Optional[Dict[str, Any]] = None+ energy_sample: Optional[Any] = None+ gpu_sample: Optional[GpuSample] = None++ if self._energy_monitor is not None:+ with self._energy_monitor.sample() as energy_sample:+ async for chunk in self._inner.stream_full(+ messages,+ model=model,+ temperature=temperature,+ max_tokens=max_tokens,+ **kwargs,+ ):+ if chunk.content:+ token_timestamps.append(time.time())+ content_chunk_count += 1+ if chunk.usage is not None:+ usage = chunk.usage+ yield chunk+ elif self._gpu_monitor is not None:+ with self._gpu_monitor.sample() as gpu_sample:+ async for chunk in self._inner.stream_full(+ messages,+ model=model,+ temperature=temperature,+ max_tokens=max_tokens,+ **kwargs,+ ):+ if chunk.content:+ token_timestamps.append(time.time())+ content_chunk_count += 1+ if chunk.usage is not None:+ usage = chunk.usage+ yield chunk+ else:+ async for chunk in self._inner.stream_full(+ messages,+ model=model,+ temperature=temperature,+ max_tokens=max_tokens,+ **kwargs,+ ):+ if chunk.content:+ token_timestamps.append(time.time())+ content_chunk_count += 1+ if chunk.usage is not None:+ usage = chunk.usage+ yield chunk++ self._publish_stream_telemetry(model=model,- temperature=temperature,- max_tokens=max_tokens,- **kwargs,- ):- yield chunk+ t0=t0,+ token_timestamps=token_timestamps,+ fallback_token_count=content_chunk_count,+ energy_sample=energy_sample,+ gpu_sample=gpu_sample,+ usage=usage,+ )def list_models(self) -> List[str]:return self._inner.list_models()diff --git a/tests/telemetry/test_instrumented_engine.py b/tests/telemetry/test_instrumented_engine.pyindex b21169f8..88a1974d 100644--- a/tests/telemetry/test_instrumented_engine.py+++ b/tests/telemetry/test_instrumented_engine.py@@ -2,12 +2,14 @@from __future__ import annotations+from contextlib import contextmanagerfrom unittest.mock import MagicMockimport pytestfrom openjarvis.core.events import EventBus, EventType-from openjarvis.core.types import Message, Role+from openjarvis.core.types import TOKEN_COUNTING_VERSION, Message, Role+from openjarvis.engine._stubs import StreamChunkfrom openjarvis.telemetry.instrumented_engine import InstrumentedEngine@@ -130,6 +132,243 @@ class TestInstrumentedEngine:ie = InstrumentedEngine(mock_engine, bus)assert ie.engine_id == "instrumented"+ @pytest.mark.asyncio+ async def test_stream_full_records_terminal_usage_and_preserves_chunks(self, bus):+ expected = [+ StreamChunk(content="Hello"),+ StreamChunk(tool_calls=[{"index": 0, "function": {"name": "lookup"}}]),+ StreamChunk(+ finish_reason="tool_calls",+ usage={+ "prompt_tokens": 7,+ "prompt_tokens_evaluated": 5,+ "completion_tokens": 3,+ "total_tokens": 10,+ },+ ),+ ]++ class StreamFullEngine:+ engine_id = "stream-full"++ async def stream_full(self, messages, *, model, **kwargs):+ for chunk in expected:+ yield chunk++ engine = InstrumentedEngine(StreamFullEngine(), bus)+ messages = [Message(role=Role.USER, content="Hi")]++ actual = [+ chunk async for chunk in engine.stream_full(messages, model="test-model")+ ]++ assert actual == expected+ assert all(+ actual_chunk is expected_chunk+ for actual_chunk, expected_chunk in zip(actual, expected)+ )+ assert [event.event_type for event in bus.history] == [+ EventType.INFERENCE_START,+ EventType.INFERENCE_END,+ EventType.TELEMETRY_RECORD,+ ]+ record = bus.history[-1].data["record"]+ assert record.engine == "stream-full"+ assert record.is_streaming is True+ assert record.prompt_tokens == 7+ assert record.prompt_tokens_evaluated == 5+ assert record.completion_tokens == 3+ assert record.total_tokens == 10+ assert record.token_counting_version == TOKEN_COUNTING_VERSION+ end_event = bus.history[-2]+ assert end_event.data["usage"]["completion_tokens"] == 3++ @pytest.mark.asyncio+ async def test_stream_full_without_usage_counts_only_content_chunks(self, bus):+ chunks = [+ StreamChunk(content="first"),+ StreamChunk(content=""),+ StreamChunk(tool_calls=[{"index": 0}]),+ StreamChunk(content="second"),+ StreamChunk(finish_reason="stop"),+ ]++ class StreamFullEngine:+ engine_id = "stream-full"++ async def stream_full(self, messages, *, model, **kwargs):+ for chunk in chunks:+ yield chunk++ engine = InstrumentedEngine(StreamFullEngine(), bus)+ messages = [Message(role=Role.USER, content="Hi")]++ actual = [+ chunk async for chunk in engine.stream_full(messages, model="test-model")+ ]++ assert actual == chunks+ assert [event.event_type for event in bus.history] == [+ EventType.INFERENCE_START,+ EventType.INFERENCE_END,+ EventType.TELEMETRY_RECORD,+ ]+ record = next(+ event.data["record"]+ for event in bus.history+ if event.event_type == EventType.TELEMETRY_RECORD+ )+ assert record.completion_tokens == 2+ assert record.prompt_tokens == 0+ assert record.total_tokens == 0+ assert record.is_streaming is True+ end_event = next(+ event+ for event in bus.history+ if event.event_type == EventType.INFERENCE_END+ )+ assert "usage" not in end_event.data++ @pytest.mark.asyncio+ async def test_stream_full_prefers_later_usage_over_earlier(self, bus):+ chunks = [+ StreamChunk(+ content="Hello",+ usage={+ "prompt_tokens": 1,+ "completion_tokens": 99,+ "total_tokens": 100,+ },+ ),+ StreamChunk(+ finish_reason="stop",+ usage={+ "prompt_tokens": 7,+ "completion_tokens": 3,+ "total_tokens": 10,+ },+ ),+ ]++ class StreamFullEngine:+ engine_id = "stream-full"++ async def stream_full(self, messages, *, model, **kwargs):+ for chunk in chunks:+ yield chunk++ engine = InstrumentedEngine(StreamFullEngine(), bus)+ messages = [Message(role=Role.USER, content="Hi")]++ actual = [+ chunk async for chunk in engine.stream_full(messages, model="test-model")+ ]++ assert actual == chunks+ record = next(+ event.data["record"]+ for event in bus.history+ if event.event_type == EventType.TELEMETRY_RECORD+ )+ assert record.prompt_tokens == 7+ assert record.completion_tokens == 3+ assert record.total_tokens == 10++ @pytest.mark.asyncio+ async def test_stream_shared_finalizer_preserves_no_usage_shape(self, bus):+ class StreamEngine:+ engine_id = "stream"++ async def stream(self, messages, *, model, **kwargs):+ for token in ["Hello", "", " world"]:+ yield token++ engine = InstrumentedEngine(StreamEngine(), bus)+ messages = [Message(role=Role.USER, content="Hi")]++ tokens = [token async for token in engine.stream(messages, model="test-model")]++ assert tokens == ["Hello", "", " world"]+ assert [event.event_type for event in bus.history] == [+ EventType.INFERENCE_START,+ EventType.INFERENCE_END,+ EventType.TELEMETRY_RECORD,+ ]+ record = bus.history[-1].data["record"]+ assert record.completion_tokens == 3+ assert record.prompt_tokens == 0+ assert record.total_tokens == 0+ assert record.is_streaming is True+ assert "usage" not in bus.history[-2].data++ @pytest.mark.asyncio+ async def test_stream_full_tool_only_records_energy(self, bus):+ class EnergySample:+ energy_joules = 4.0+ mean_power_watts = 8.0+ mean_utilization_pct = 25.0+ peak_memory_used_gb = 1.0+ mean_temperature_c = 40.0+ energy_method = "test"+ vendor = "test"+ cpu_energy_joules = 1.0+ gpu_energy_joules = 3.0+ dram_energy_joules = 0.0+ ane_energy_joules = 0.0+ soc_energy_joules = 4.0+ basis = "soc"++ class EnergyMonitor:+ active = False++ @contextmanager+ def sample(self):+ self.active = True+ try:+ yield EnergySample()+ finally:+ self.active = False++ monitor = EnergyMonitor()++ class StreamFullEngine:+ engine_id = "stream-full"++ async def stream_full(self, messages, *, model, **kwargs):+ assert monitor.active is True+ yield StreamChunk(tool_calls=[{"index": 0}])+ assert monitor.active is True+ yield StreamChunk(+ finish_reason="tool_calls",+ usage={+ "prompt_tokens": 4,+ "completion_tokens": 0,+ "total_tokens": 4,+ },+ )++ engine = InstrumentedEngine(+ StreamFullEngine(),+ bus,+ energy_monitor=monitor,+ )+ messages = [Message(role=Role.USER, content="Use a tool")]++ chunks = [+ chunk async for chunk in engine.stream_full(messages, model="test-model")+ ]++ assert len(chunks) == 2+ assert monitor.active is False+ record = next(+ event.data["record"]+ for event in bus.history+ if event.event_type == EventType.TELEMETRY_RECORD+ )+ assert record.prompt_tokens == 4+ assert record.completion_tokens == 0+ assert record.energy_joules == 4.0+class TestTokensPerJoule:def test_tokens_per_joule_zero_without_energy(self, mock_engine, bus):