Skip to content

Personal AI, On Personal Devices

Python

fix(telemetry): record successful stream_full inferences (#1020)

Johnny C Wattccce49692ca0
diff --git a/src/openjarvis/telemetry/instrumented_engine.py b/src/openjarvis/telemetry/instrumented_engine.py
index 129bdb3a..55cf5086 100644
--- a/src/openjarvis/telemetry/instrumented_engine.py
+++ b/src/openjarvis/telemetry/instrumented_engine.py
@@ -315,79 +315,55 @@ class InstrumentedEngine(InferenceEngine):
return result
- async def stream(
+ def _publish_stream_telemetry(
self,
- messages: Sequence[Message],
*,
- model: str = "",
- temperature: float = 0.7,
- max_tokens: int = 1024,
- **kwargs: Any,
- ) -> Any:
- """Stream with per-token timing and full telemetry recording."""
- self._bus.publish(
- EventType.INFERENCE_START,
- {
- "model": model,
- "message_count": len(messages),
- },
- )
-
- t0 = time.time()
- token_timestamps: list[float] = []
- token_count = 0
-
- energy_sample: Optional[Any] = None
- gpu_sample: Optional[GpuSample] = None
-
- if self._energy_monitor is not None:
- with self._energy_monitor.sample() as energy_sample:
- async for token in self._inner.stream(
- messages,
- model=model,
- temperature=temperature,
- max_tokens=max_tokens,
- **kwargs,
- ):
- token_timestamps.append(time.time())
- token_count += 1
- yield token
- elif self._gpu_monitor is not None:
- with self._gpu_monitor.sample() as gpu_sample:
- async for token in self._inner.stream(
- messages,
- model=model,
- temperature=temperature,
- max_tokens=max_tokens,
- **kwargs,
- ):
- token_timestamps.append(time.time())
- token_count += 1
- yield token
+ model: str,
+ t0: float,
+ token_timestamps: list[float],
+ fallback_token_count: int,
+ energy_sample: Optional[Any],
+ gpu_sample: Optional[GpuSample],
+ usage: Optional[Dict[str, Any]] = None,
+ ) -> None:
+ """Publish telemetry after a streaming call completes successfully."""
+ latency = time.time() - t0
+ usage_data = usage or {}
+
+ if usage is None:
+ prompt_tokens = 0
+ prompt_tokens_evaluated = 0
+ completion_tokens = fallback_token_count
+ # Preserve stream()'s existing record shape when provider usage is
+ # unavailable. Rich streams with usage populate the exact total.
+ total_tokens = 0
else:
- async for token in self._inner.stream(
- messages,
- model=model,
- temperature=temperature,
- max_tokens=max_tokens,
- **kwargs,
- ):
- token_timestamps.append(time.time())
- token_count += 1
- yield token
+ prompt_tokens = usage_data.get("prompt_tokens", 0) or 0
+ prompt_tokens_evaluated = (
+ usage_data.get("prompt_tokens_evaluated") or prompt_tokens
+ )
+ reported_completion = usage_data.get("completion_tokens")
+ completion_tokens = (
+ fallback_token_count
+ if reported_completion is None
+ else reported_completion
+ )
+ reported_total = usage_data.get("total_tokens")
+ total_tokens = (
+ prompt_tokens + completion_tokens
+ if reported_total is None
+ else reported_total
+ )
- latency = time.time() - t0
ttft = token_timestamps[0] - t0 if token_timestamps else 0.0
- throughput = token_count / latency if latency > 0 else 0.0
+ throughput = completion_tokens / latency if latency > 0 else 0.0
- # Compute ITL from consecutive timestamps
itl_values_ms = [
(token_timestamps[i] - token_timestamps[i - 1]) * 1000
for i in range(1, len(token_timestamps))
]
itl_stats = _compute_itl_stats(itl_values_ms)
- # Energy / GPU metrics from sample
energy_joules = 0.0
power_watts = 0.0
gpu_utilization_pct = 0.0
@@ -424,20 +400,16 @@ class InstrumentedEngine(InferenceEngine):
gpu_temperature_c = gpu_sample.mean_temperature_c
energy_method = "polling"
energy_vendor = "nvidia"
- # The legacy monitor reads one GPU rail and nothing else.
gpu_energy_joules = gpu_sample.energy_joules
soc_energy_joules = gpu_sample.energy_joules
energy_basis = BASIS_GPU
prefill_latency = ttft if ttft > 0 else 0.0
-
- # Derived metrics
energy_per_output_token = (
- energy_joules / token_count if token_count > 0 else 0.0
+ energy_joules / completion_tokens if completion_tokens > 0 else 0.0
)
throughput_per_watt = throughput / power_watts if power_watts > 0 else 0.0
- # Phase energy split
decode_latency = latency - prefill_latency if prefill_latency > 0 else 0.0
prefill_energy = 0.0
decode_energy = 0.0
@@ -446,19 +418,20 @@ class InstrumentedEngine(InferenceEngine):
prefill_energy = energy_joules * prefill_frac
decode_energy = energy_joules * (1.0 - prefill_frac)
- # Per-inference efficiency
tokens_per_joule = (
- token_count / energy_joules
- if energy_joules > 0 and token_count > 0
+ completion_tokens / energy_joules
+ if energy_joules > 0 and completion_tokens > 0
else 0.0
)
engine_id = getattr(self._inner, "engine_id", "unknown")
-
record = TelemetryRecord(
timestamp=t0,
model_id=model,
- completion_tokens=token_count,
+ prompt_tokens=prompt_tokens,
+ prompt_tokens_evaluated=prompt_tokens_evaluated,
+ completion_tokens=completion_tokens,
+ total_tokens=total_tokens,
latency_seconds=latency,
ttft=ttft,
throughput_tok_per_sec=throughput,
@@ -490,7 +463,6 @@ class InstrumentedEngine(InferenceEngine):
soc_energy_joules=soc_energy_joules,
energy_basis=energy_basis,
tokens_per_joule=tokens_per_joule,
- # Stamp the methodology version on streaming records too.
token_counting_version=TOKEN_COUNTING_VERSION,
)
@@ -499,7 +471,7 @@ class InstrumentedEngine(InferenceEngine):
"latency": latency,
"ttft": ttft,
"throughput_tok_per_sec": throughput,
- "completion_tokens": token_count,
+ "completion_tokens": completion_tokens,
"is_streaming": True,
"mean_itl_ms": itl_stats["mean"],
"median_itl_ms": itl_stats["median"],
@@ -509,10 +481,82 @@ class InstrumentedEngine(InferenceEngine):
"energy_method": energy_method,
"energy_vendor": energy_vendor,
}
+ if usage is not None:
+ event_data["usage"] = usage_data
self._bus.publish(EventType.INFERENCE_END, event_data)
self._bus.publish(EventType.TELEMETRY_RECORD, {"record": record})
+ async def stream(
+ self,
+ messages: Sequence[Message],
+ *,
+ model: str = "",
+ temperature: float = 0.7,
+ max_tokens: int = 1024,
+ **kwargs: Any,
+ ) -> Any:
+ """Stream with per-token timing and full telemetry recording."""
+ self._bus.publish(
+ EventType.INFERENCE_START,
+ {
+ "model": model,
+ "message_count": len(messages),
+ },
+ )
+
+ t0 = time.time()
+ token_timestamps: list[float] = []
+ token_count = 0
+
+ energy_sample: Optional[Any] = None
+ gpu_sample: Optional[GpuSample] = None
+
+ if self._energy_monitor is not None:
+ with self._energy_monitor.sample() as energy_sample:
+ async for token in self._inner.stream(
+ messages,
+ model=model,
+ temperature=temperature,
+ max_tokens=max_tokens,
+ **kwargs,
+ ):
+ token_timestamps.append(time.time())
+ token_count += 1
+ yield token
+ elif self._gpu_monitor is not None:
+ with self._gpu_monitor.sample() as gpu_sample:
+ async for token in self._inner.stream(
+ messages,
+ model=model,
+ temperature=temperature,
+ max_tokens=max_tokens,
+ **kwargs,
+ ):
+ token_timestamps.append(time.time())
+ token_count += 1
+ yield token
+ else:
+ async for token in self._inner.stream(
+ messages,
+ model=model,
+ temperature=temperature,
+ max_tokens=max_tokens,
+ **kwargs,
+ ):
+ token_timestamps.append(time.time())
+ token_count += 1
+ yield token
+
+ self._publish_stream_telemetry(
+ model=model,
+ t0=t0,
+ token_timestamps=token_timestamps,
+ fallback_token_count=token_count,
+ energy_sample=energy_sample,
+ gpu_sample=gpu_sample,
+ )
+
async def stream_full(
self,
messages: Sequence[Message],
@@ -522,15 +566,80 @@ class InstrumentedEngine(InferenceEngine):
max_tokens: int = 1024,
**kwargs: Any,
) -> AsyncIterator["StreamChunk"]:
- """Delegate to inner engine's stream_full for tool-call support."""
- async for chunk in self._inner.stream_full(
- messages,
+ """Stream rich chunks with timing, usage, and energy telemetry.
+
+ Telemetry is published only after the inner stream completes
+ successfully.
+ """
+ self._bus.publish(
+ EventType.INFERENCE_START,
+ {
+ "model": model,
+ "message_count": len(messages),
+ },
+ )
+
+ t0 = time.time()
+ token_timestamps: list[float] = []
+ content_chunk_count = 0
+ usage: Optional[Dict[str, Any]] = None
+ energy_sample: Optional[Any] = None
+ gpu_sample: Optional[GpuSample] = None
+
+ if self._energy_monitor is not None:
+ with self._energy_monitor.sample() as energy_sample:
+ async for chunk in self._inner.stream_full(
+ messages,
+ model=model,
+ temperature=temperature,
+ max_tokens=max_tokens,
+ **kwargs,
+ ):
+ if chunk.content:
+ token_timestamps.append(time.time())
+ content_chunk_count += 1
+ if chunk.usage is not None:
+ usage = chunk.usage
+ yield chunk
+ elif self._gpu_monitor is not None:
+ with self._gpu_monitor.sample() as gpu_sample:
+ async for chunk in self._inner.stream_full(
+ messages,
+ model=model,
+ temperature=temperature,
+ max_tokens=max_tokens,
+ **kwargs,
+ ):
+ if chunk.content:
+ token_timestamps.append(time.time())
+ content_chunk_count += 1
+ if chunk.usage is not None:
+ usage = chunk.usage
+ yield chunk
+ else:
+ async for chunk in self._inner.stream_full(
+ messages,
+ model=model,
+ temperature=temperature,
+ max_tokens=max_tokens,
+ **kwargs,
+ ):
+ if chunk.content:
+ token_timestamps.append(time.time())
+ content_chunk_count += 1
+ if chunk.usage is not None:
+ usage = chunk.usage
+ yield chunk
+
+ self._publish_stream_telemetry(
model=model,
- temperature=temperature,
- max_tokens=max_tokens,
- **kwargs,
- ):
- yield chunk
+ t0=t0,
+ token_timestamps=token_timestamps,
+ fallback_token_count=content_chunk_count,
+ energy_sample=energy_sample,
+ gpu_sample=gpu_sample,
+ usage=usage,
+ )
def list_models(self) -> List[str]:
return self._inner.list_models()
diff --git a/tests/telemetry/test_instrumented_engine.py b/tests/telemetry/test_instrumented_engine.py
index b21169f8..88a1974d 100644
--- a/tests/telemetry/test_instrumented_engine.py
+++ b/tests/telemetry/test_instrumented_engine.py
@@ -2,12 +2,14 @@
from __future__ import annotations
+from contextlib import contextmanager
from unittest.mock import MagicMock
import pytest
from openjarvis.core.events import EventBus, EventType
-from openjarvis.core.types import Message, Role
+from openjarvis.core.types import TOKEN_COUNTING_VERSION, Message, Role
+from openjarvis.engine._stubs import StreamChunk
from openjarvis.telemetry.instrumented_engine import InstrumentedEngine
@@ -130,6 +132,243 @@ class TestInstrumentedEngine:
ie = InstrumentedEngine(mock_engine, bus)
assert ie.engine_id == "instrumented"
+ @pytest.mark.asyncio
+ async def test_stream_full_records_terminal_usage_and_preserves_chunks(self, bus):
+ expected = [
+ StreamChunk(content="Hello"),
+ StreamChunk(tool_calls=[{"index": 0, "function": {"name": "lookup"}}]),
+ StreamChunk(
+ finish_reason="tool_calls",
+ usage={
+ "prompt_tokens": 7,
+ "prompt_tokens_evaluated": 5,
+ "completion_tokens": 3,
+ "total_tokens": 10,
+ },
+ ),
+ ]
+
+ class StreamFullEngine:
+ engine_id = "stream-full"
+
+ async def stream_full(self, messages, *, model, **kwargs):
+ for chunk in expected:
+ yield chunk
+
+ engine = InstrumentedEngine(StreamFullEngine(), bus)
+ messages = [Message(role=Role.USER, content="Hi")]
+
+ actual = [
+ chunk async for chunk in engine.stream_full(messages, model="test-model")
+ ]
+
+ assert actual == expected
+ assert all(
+ actual_chunk is expected_chunk
+ for actual_chunk, expected_chunk in zip(actual, expected)
+ )
+ assert [event.event_type for event in bus.history] == [
+ EventType.INFERENCE_START,
+ EventType.INFERENCE_END,
+ EventType.TELEMETRY_RECORD,
+ ]
+ record = bus.history[-1].data["record"]
+ assert record.engine == "stream-full"
+ assert record.is_streaming is True
+ assert record.prompt_tokens == 7
+ assert record.prompt_tokens_evaluated == 5
+ assert record.completion_tokens == 3
+ assert record.total_tokens == 10
+ assert record.token_counting_version == TOKEN_COUNTING_VERSION
+ end_event = bus.history[-2]
+ assert end_event.data["usage"]["completion_tokens"] == 3
+
+ @pytest.mark.asyncio
+ async def test_stream_full_without_usage_counts_only_content_chunks(self, bus):
+ chunks = [
+ StreamChunk(content="first"),
+ StreamChunk(content=""),
+ StreamChunk(tool_calls=[{"index": 0}]),
+ StreamChunk(content="second"),
+ StreamChunk(finish_reason="stop"),
+ ]
+
+ class StreamFullEngine:
+ engine_id = "stream-full"
+
+ async def stream_full(self, messages, *, model, **kwargs):
+ for chunk in chunks:
+ yield chunk
+
+ engine = InstrumentedEngine(StreamFullEngine(), bus)
+ messages = [Message(role=Role.USER, content="Hi")]
+
+ actual = [
+ chunk async for chunk in engine.stream_full(messages, model="test-model")
+ ]
+
+ assert actual == chunks
+ assert [event.event_type for event in bus.history] == [
+ EventType.INFERENCE_START,
+ EventType.INFERENCE_END,
+ EventType.TELEMETRY_RECORD,
+ ]
+ record = next(
+ event.data["record"]
+ for event in bus.history
+ if event.event_type == EventType.TELEMETRY_RECORD
+ )
+ assert record.completion_tokens == 2
+ assert record.prompt_tokens == 0
+ assert record.total_tokens == 0
+ assert record.is_streaming is True
+ end_event = next(
+ event
+ for event in bus.history
+ if event.event_type == EventType.INFERENCE_END
+ )
+ assert "usage" not in end_event.data
+
+ @pytest.mark.asyncio
+ async def test_stream_full_prefers_later_usage_over_earlier(self, bus):
+ chunks = [
+ StreamChunk(
+ content="Hello",
+ usage={
+ "prompt_tokens": 1,
+ "completion_tokens": 99,
+ "total_tokens": 100,
+ },
+ ),
+ StreamChunk(
+ finish_reason="stop",
+ usage={
+ "prompt_tokens": 7,
+ "completion_tokens": 3,
+ "total_tokens": 10,
+ },
+ ),
+ ]
+
+ class StreamFullEngine:
+ engine_id = "stream-full"
+
+ async def stream_full(self, messages, *, model, **kwargs):
+ for chunk in chunks:
+ yield chunk
+
+ engine = InstrumentedEngine(StreamFullEngine(), bus)
+ messages = [Message(role=Role.USER, content="Hi")]
+
+ actual = [
+ chunk async for chunk in engine.stream_full(messages, model="test-model")
+ ]
+
+ assert actual == chunks
+ record = next(
+ event.data["record"]
+ for event in bus.history
+ if event.event_type == EventType.TELEMETRY_RECORD
+ )
+ assert record.prompt_tokens == 7
+ assert record.completion_tokens == 3
+ assert record.total_tokens == 10
+
+ @pytest.mark.asyncio
+ async def test_stream_shared_finalizer_preserves_no_usage_shape(self, bus):
+ class StreamEngine:
+ engine_id = "stream"
+
+ async def stream(self, messages, *, model, **kwargs):
+ for token in ["Hello", "", " world"]:
+ yield token
+
+ engine = InstrumentedEngine(StreamEngine(), bus)
+ messages = [Message(role=Role.USER, content="Hi")]
+
+ tokens = [token async for token in engine.stream(messages, model="test-model")]
+
+ assert tokens == ["Hello", "", " world"]
+ assert [event.event_type for event in bus.history] == [
+ EventType.INFERENCE_START,
+ EventType.INFERENCE_END,
+ EventType.TELEMETRY_RECORD,
+ ]
+ record = bus.history[-1].data["record"]
+ assert record.completion_tokens == 3
+ assert record.prompt_tokens == 0
+ assert record.total_tokens == 0
+ assert record.is_streaming is True
+ assert "usage" not in bus.history[-2].data
+
+ @pytest.mark.asyncio
+ async def test_stream_full_tool_only_records_energy(self, bus):
+ class EnergySample:
+ energy_joules = 4.0
+ mean_power_watts = 8.0
+ mean_utilization_pct = 25.0
+ peak_memory_used_gb = 1.0
+ mean_temperature_c = 40.0
+ energy_method = "test"
+ vendor = "test"
+ cpu_energy_joules = 1.0
+ gpu_energy_joules = 3.0
+ dram_energy_joules = 0.0
+ ane_energy_joules = 0.0
+ soc_energy_joules = 4.0
+ basis = "soc"
+
+ class EnergyMonitor:
+ active = False
+
+ @contextmanager
+ def sample(self):
+ self.active = True
+ try:
+ yield EnergySample()
+ finally:
+ self.active = False
+
+ monitor = EnergyMonitor()
+
+ class StreamFullEngine:
+ engine_id = "stream-full"
+
+ async def stream_full(self, messages, *, model, **kwargs):
+ assert monitor.active is True
+ yield StreamChunk(tool_calls=[{"index": 0}])
+ assert monitor.active is True
+ yield StreamChunk(
+ finish_reason="tool_calls",
+ usage={
+ "prompt_tokens": 4,
+ "completion_tokens": 0,
+ "total_tokens": 4,
+ },
+ )
+
+ engine = InstrumentedEngine(
+ StreamFullEngine(),
+ bus,
+ energy_monitor=monitor,
+ )
+ messages = [Message(role=Role.USER, content="Use a tool")]
+
+ chunks = [
+ chunk async for chunk in engine.stream_full(messages, model="test-model")
+ ]
+
+ assert len(chunks) == 2
+ assert monitor.active is False
+ record = next(
+ event.data["record"]
+ for event in bus.history
+ if event.event_type == EventType.TELEMETRY_RECORD
+ )
+ assert record.prompt_tokens == 4
+ assert record.completion_tokens == 0
+ assert record.energy_joules == 4.0
+
class TestTokensPerJoule:
def test_tokens_per_joule_zero_without_energy(self, mock_engine, bus):