Skip to content

Personal AI, On Personal Devices

Python
24.3 KB
Raw Blame
"""Transparent telemetry wrapper for inference engines."""

from __future__ import annotations

import statistics
import time
from collections.abc import AsyncIterator
from typing import Any, Dict, List, Optional, Sequence

from openjarvis.core.events import EventBus, EventType
from openjarvis.core.types import TOKEN_COUNTING_VERSION, Message, TelemetryRecord
from openjarvis.engine._stubs import InferenceEngine, StreamChunk
from openjarvis.telemetry.energy_monitor import BASIS_GPU
from openjarvis.telemetry.gpu_monitor import GpuSample

# ---------------------------------------------------------------------------
# ITL helpers
# ---------------------------------------------------------------------------


def _percentile(data: list[float], p: float) -> float:
    """Compute the p-th percentile using linear interpolation."""
    sorted_data = sorted(data)
    k = (len(sorted_data) - 1) * p
    f = int(k)
    c = f + 1
    if c >= len(sorted_data):
        return sorted_data[-1]
    return sorted_data[f] + (k - f) * (sorted_data[c] - sorted_data[f])


def _compute_itl_stats(itl_values_ms: list[float]) -> dict:
    """Compute ITL summary statistics from a list of inter-token latencies in ms."""
    if not itl_values_ms:
        return {
            "mean": 0.0,
            "median": 0.0,
            "p90": 0.0,
            "p95": 0.0,
            "p99": 0.0,
            "std": 0.0,
        }
    return {
        "mean": statistics.mean(itl_values_ms),
        "median": statistics.median(itl_values_ms),
        "p90": _percentile(itl_values_ms, 0.90),
        "p95": _percentile(itl_values_ms, 0.95),
        "p99": _percentile(itl_values_ms, 0.99),
        "std": statistics.stdev(itl_values_ms) if len(itl_values_ms) > 1 else 0.0,
    }


class InstrumentedEngine(InferenceEngine):
    """Transparent wrapper that records telemetry around engine calls.

    Agents call ``engine.generate()`` normally -- they don't know
    about telemetry.  The wrapper publishes ``INFERENCE_START``,
    ``INFERENCE_END``, and ``TELEMETRY_RECORD`` events on the bus.

    If an ``energy_monitor`` is provided (new multi-vendor
    :class:`~openjarvis.telemetry.energy_monitor.EnergyMonitor`), it is
    preferred over the legacy ``gpu_monitor`` for energy measurement.
    """

    engine_id = "instrumented"

    def __init__(
        self,
        engine: InferenceEngine,
        bus: EventBus,
        gpu_monitor: Optional[Any] = None,
        energy_monitor: Optional[Any] = None,
    ) -> None:
        self._inner = engine
        self._bus = bus
        self._gpu_monitor = gpu_monitor
        self._energy_monitor = energy_monitor
        self._publishes_events = True

    def generate(
        self,
        messages: Sequence[Message],
        *,
        model: str = "",
        temperature: float = 0.7,
        max_tokens: int = 1024,
        **kwargs: Any,
    ) -> Dict[str, Any]:
        """Generate with telemetry recording."""
        self._bus.publish(
            EventType.INFERENCE_START,
            {
                "model": model,
                "message_count": len(messages),
            },
        )

        gpu_sample: Optional[GpuSample] = None
        energy_sample: Optional[Any] = None
        t0 = time.time()

        # Prefer EnergyMonitor over legacy GpuMonitor
        if self._energy_monitor is not None:
            with self._energy_monitor.sample() as energy_sample:
                result = self._inner.generate(
                    messages,
                    model=model,
                    temperature=temperature,
                    max_tokens=max_tokens,
                    **kwargs,
                )
        elif self._gpu_monitor is not None:
            with self._gpu_monitor.sample() as gpu_sample:
                result = self._inner.generate(
                    messages,
                    model=model,
                    temperature=temperature,
                    max_tokens=max_tokens,
                    **kwargs,
                )
        else:
            result = self._inner.generate(
                messages,
                model=model,
                temperature=temperature,
                max_tokens=max_tokens,
                **kwargs,
            )

        latency = time.time() - t0

        usage = result.get("usage", {})
        completion_tokens = usage.get("completion_tokens", 0)
        prompt_tokens_evaluated = usage.get("prompt_tokens_evaluated", 0)
        ttft = result.get("ttft", 0.0)
        throughput = completion_tokens / latency if latency > 0 else 0.0

        # Energy / GPU metrics from sample
        energy_joules = 0.0
        power_watts = 0.0
        gpu_utilization_pct = 0.0
        gpu_memory_used_gb = 0.0
        gpu_temperature_c = 0.0
        prefill_latency = 0.0
        energy_method = ""
        energy_vendor = ""
        cpu_energy_joules = 0.0
        gpu_energy_joules = 0.0
        dram_energy_joules = 0.0
        ane_energy_joules = 0.0
        soc_energy_joules = 0.0
        energy_basis = ""

        if energy_sample is not None:
            # New multi-vendor EnergyMonitor path
            energy_joules = energy_sample.energy_joules
            power_watts = energy_sample.mean_power_watts
            gpu_utilization_pct = energy_sample.mean_utilization_pct
            gpu_memory_used_gb = energy_sample.peak_memory_used_gb
            gpu_temperature_c = energy_sample.mean_temperature_c
            energy_method = energy_sample.energy_method
            energy_vendor = energy_sample.vendor
            cpu_energy_joules = energy_sample.cpu_energy_joules
            gpu_energy_joules = energy_sample.gpu_energy_joules
            dram_energy_joules = energy_sample.dram_energy_joules
            ane_energy_joules = energy_sample.ane_energy_joules
            soc_energy_joules = energy_sample.soc_energy_joules
            energy_basis = energy_sample.basis
        elif gpu_sample is not None:
            # Legacy GpuMonitor path
            energy_joules = gpu_sample.energy_joules
            power_watts = gpu_sample.mean_power_watts
            gpu_utilization_pct = gpu_sample.mean_utilization_pct
            gpu_memory_used_gb = gpu_sample.peak_memory_used_gb
            gpu_temperature_c = gpu_sample.mean_temperature_c
            energy_method = "polling"
            energy_vendor = "nvidia"
            # The legacy monitor reads one GPU rail and nothing else.
            gpu_energy_joules = gpu_sample.energy_joules
            soc_energy_joules = gpu_sample.energy_joules
            energy_basis = BASIS_GPU

        if ttft > 0:
            prefill_latency = ttft

        # --- Tier 1: Derived metrics ---
        energy_per_output_token = (
            energy_joules / completion_tokens if completion_tokens > 0 else 0.0
        )
        throughput_per_watt = throughput / power_watts if power_watts > 0 else 0.0

        # --- Tier 2.1: Phase energy split ---
        decode_latency = latency - prefill_latency if prefill_latency > 0 else 0.0
        prefill_energy = 0.0
        decode_energy = 0.0
        if energy_joules > 0 and prefill_latency > 0 and latency > 0:
            prefill_frac = prefill_latency / latency
            prefill_energy = energy_joules * prefill_frac
            decode_energy = energy_joules * (1.0 - prefill_frac)

        # --- Tier 3: Non-streaming mean ITL approximation ---
        mean_itl_ms = (
            (decode_latency / completion_tokens) * 1000
            if completion_tokens > 0 and decode_latency > 0
            else 0.0
        )

        # --- Tier 4: Per-inference efficiency ---
        tokens_per_joule = (
            completion_tokens / energy_joules
            if energy_joules > 0 and completion_tokens > 0
            else 0.0
        )

        engine_id = getattr(self._inner, "engine_id", "unknown")

        prompt_tok = usage.get("prompt_tokens", 0)
        record = TelemetryRecord(
            timestamp=t0,
            model_id=model,
            prompt_tokens=prompt_tok,
            prompt_tokens_evaluated=prompt_tokens_evaluated or prompt_tok,
            completion_tokens=completion_tokens,
            total_tokens=prompt_tok + completion_tokens,
            latency_seconds=latency,
            cost_usd=result.get("cost_usd", 0.0),
            ttft=ttft,
            throughput_tok_per_sec=throughput,
            energy_per_output_token_joules=energy_per_output_token,
            throughput_per_watt=throughput_per_watt,
            energy_joules=energy_joules,
            power_watts=power_watts,
            gpu_utilization_pct=gpu_utilization_pct,
            gpu_memory_used_gb=gpu_memory_used_gb,
            gpu_temperature_c=gpu_temperature_c,
            prefill_latency_seconds=prefill_latency,
            decode_latency_seconds=decode_latency,
            prefill_energy_joules=prefill_energy,
            decode_energy_joules=decode_energy,
            mean_itl_ms=mean_itl_ms,
            engine=engine_id,
            energy_method=energy_method,
            energy_vendor=energy_vendor,
            cpu_energy_joules=cpu_energy_joules,
            gpu_energy_joules=gpu_energy_joules,
            dram_energy_joules=dram_energy_joules,
            ane_energy_joules=ane_energy_joules,
            soc_energy_joules=soc_energy_joules,
            energy_basis=energy_basis,
            tokens_per_joule=tokens_per_joule,
            # Stamp every new record with the current methodology version
            # so the leaderboard aggregator can drop legacy rows (those
            # left NULL by pre-fix builds) cleanly. Source of truth for
            # the integer is `server/savings.py`.
            token_counting_version=TOKEN_COUNTING_VERSION,
        )

        event_data = {
            "model": model,
            "latency": latency,
            "usage": usage,
            "ttft": ttft,
            "throughput_tok_per_sec": throughput,
            "energy_per_output_token_joules": energy_per_output_token,
            "throughput_per_watt": throughput_per_watt,
            "energy_joules": energy_joules,
            "power_watts": power_watts,
            "gpu_utilization_pct": gpu_utilization_pct,
            "gpu_memory_used_gb": gpu_memory_used_gb,
            "gpu_temperature_c": gpu_temperature_c,
            "prefill_latency_seconds": prefill_latency,
            "decode_latency_seconds": decode_latency,
            "prefill_energy_joules": prefill_energy,
            "decode_energy_joules": decode_energy,
            "mean_itl_ms": mean_itl_ms,
            "energy_method": energy_method,
            "energy_vendor": energy_vendor,
            # Rich trace data: model response content + structured blocks
            "content": result.get("content", ""),
            "tool_calls": result.get("tool_calls", []),
            "tool_results": result.get("tool_results", []),
            "content_blocks": result.get("content_blocks", []),
            "finish_reason": result.get("finish_reason", ""),
        }

        self._bus.publish(EventType.INFERENCE_END, event_data)
        self._bus.publish(EventType.TELEMETRY_RECORD, {"record": record})

        # Inject telemetry dict into result for downstream consumers (eval backend)
        result["_telemetry"] = {
            "latency": latency,
            "ttft": ttft,
            "throughput_tok_per_sec": throughput,
            "energy_per_output_token_joules": energy_per_output_token,
            "throughput_per_watt": throughput_per_watt,
            "energy_joules": energy_joules,
            "power_watts": power_watts,
            "gpu_utilization_pct": gpu_utilization_pct,
            "gpu_memory_used_gb": gpu_memory_used_gb,
            "gpu_temperature_c": gpu_temperature_c,
            "prefill_latency_seconds": prefill_latency,
            "decode_latency_seconds": decode_latency,
            "prefill_energy_joules": prefill_energy,
            "decode_energy_joules": decode_energy,
            "mean_itl_ms": mean_itl_ms,
            "energy_method": energy_method,
            "energy_vendor": energy_vendor,
            "cpu_energy_joules": cpu_energy_joules,
            "gpu_energy_joules": gpu_energy_joules,
            "dram_energy_joules": dram_energy_joules,
            "ane_energy_joules": ane_energy_joules,
            "soc_energy_joules": soc_energy_joules,
            "energy_basis": energy_basis,
        }

        return result

    def _publish_stream_telemetry(
        self,
        *,
        model: str,
        t0: float,
        token_timestamps: list[float],
        fallback_token_count: int,
        energy_sample: Optional[Any],
        gpu_sample: Optional[GpuSample],
        usage: Optional[Dict[str, Any]] = None,
    ) -> None:
        """Publish telemetry after a streaming call completes successfully."""
        latency = time.time() - t0
        usage_data = usage or {}

        if usage is None:
            prompt_tokens = 0
            prompt_tokens_evaluated = 0
            completion_tokens = fallback_token_count
            # Preserve stream()'s existing record shape when provider usage is
            # unavailable. Rich streams with usage populate the exact total.
            total_tokens = 0
        else:
            prompt_tokens = usage_data.get("prompt_tokens", 0) or 0
            prompt_tokens_evaluated = (
                usage_data.get("prompt_tokens_evaluated") or prompt_tokens
            )
            reported_completion = usage_data.get("completion_tokens")
            completion_tokens = (
                fallback_token_count
                if reported_completion is None
                else reported_completion
            )
            reported_total = usage_data.get("total_tokens")
            total_tokens = (
                prompt_tokens + completion_tokens
                if reported_total is None
                else reported_total
            )

        ttft = token_timestamps[0] - t0 if token_timestamps else 0.0
        throughput = completion_tokens / latency if latency > 0 else 0.0

        itl_values_ms = [
            (token_timestamps[i] - token_timestamps[i - 1]) * 1000
            for i in range(1, len(token_timestamps))
        ]
        itl_stats = _compute_itl_stats(itl_values_ms)

        energy_joules = 0.0
        power_watts = 0.0
        gpu_utilization_pct = 0.0
        gpu_memory_used_gb = 0.0
        gpu_temperature_c = 0.0
        energy_method = ""
        energy_vendor = ""
        cpu_energy_joules = 0.0
        gpu_energy_joules = 0.0
        dram_energy_joules = 0.0
        ane_energy_joules = 0.0
        soc_energy_joules = 0.0
        energy_basis = ""

        if energy_sample is not None:
            energy_joules = energy_sample.energy_joules
            power_watts = energy_sample.mean_power_watts
            gpu_utilization_pct = energy_sample.mean_utilization_pct
            gpu_memory_used_gb = energy_sample.peak_memory_used_gb
            gpu_temperature_c = energy_sample.mean_temperature_c
            energy_method = energy_sample.energy_method
            energy_vendor = energy_sample.vendor
            cpu_energy_joules = energy_sample.cpu_energy_joules
            gpu_energy_joules = energy_sample.gpu_energy_joules
            dram_energy_joules = energy_sample.dram_energy_joules
            ane_energy_joules = energy_sample.ane_energy_joules
            soc_energy_joules = energy_sample.soc_energy_joules
            energy_basis = energy_sample.basis
        elif gpu_sample is not None:
            energy_joules = gpu_sample.energy_joules
            power_watts = gpu_sample.mean_power_watts
            gpu_utilization_pct = gpu_sample.mean_utilization_pct
            gpu_memory_used_gb = gpu_sample.peak_memory_used_gb
            gpu_temperature_c = gpu_sample.mean_temperature_c
            energy_method = "polling"
            energy_vendor = "nvidia"
            gpu_energy_joules = gpu_sample.energy_joules
            soc_energy_joules = gpu_sample.energy_joules
            energy_basis = BASIS_GPU

        prefill_latency = ttft if ttft > 0 else 0.0
        energy_per_output_token = (
            energy_joules / completion_tokens if completion_tokens > 0 else 0.0
        )
        throughput_per_watt = throughput / power_watts if power_watts > 0 else 0.0

        decode_latency = latency - prefill_latency if prefill_latency > 0 else 0.0
        prefill_energy = 0.0
        decode_energy = 0.0
        if energy_joules > 0 and prefill_latency > 0 and latency > 0:
            prefill_frac = prefill_latency / latency
            prefill_energy = energy_joules * prefill_frac
            decode_energy = energy_joules * (1.0 - prefill_frac)

        tokens_per_joule = (
            completion_tokens / energy_joules
            if energy_joules > 0 and completion_tokens > 0
            else 0.0
        )

        engine_id = getattr(self._inner, "engine_id", "unknown")
        record = TelemetryRecord(
            timestamp=t0,
            model_id=model,
            prompt_tokens=prompt_tokens,
            prompt_tokens_evaluated=prompt_tokens_evaluated,
            completion_tokens=completion_tokens,
            total_tokens=total_tokens,
            latency_seconds=latency,
            ttft=ttft,
            throughput_tok_per_sec=throughput,
            energy_per_output_token_joules=energy_per_output_token,
            throughput_per_watt=throughput_per_watt,
            energy_joules=energy_joules,
            power_watts=power_watts,
            gpu_utilization_pct=gpu_utilization_pct,
            gpu_memory_used_gb=gpu_memory_used_gb,
            gpu_temperature_c=gpu_temperature_c,
            prefill_latency_seconds=prefill_latency,
            decode_latency_seconds=decode_latency,
            prefill_energy_joules=prefill_energy,
            decode_energy_joules=decode_energy,
            mean_itl_ms=itl_stats["mean"],
            median_itl_ms=itl_stats["median"],
            p90_itl_ms=itl_stats["p90"],
            p95_itl_ms=itl_stats["p95"],
            p99_itl_ms=itl_stats["p99"],
            std_itl_ms=itl_stats["std"],
            is_streaming=True,
            engine=engine_id,
            energy_method=energy_method,
            energy_vendor=energy_vendor,
            cpu_energy_joules=cpu_energy_joules,
            gpu_energy_joules=gpu_energy_joules,
            dram_energy_joules=dram_energy_joules,
            ane_energy_joules=ane_energy_joules,
            soc_energy_joules=soc_energy_joules,
            energy_basis=energy_basis,
            tokens_per_joule=tokens_per_joule,
            token_counting_version=TOKEN_COUNTING_VERSION,
        )

        event_data = {
            "model": model,
            "latency": latency,
            "ttft": ttft,
            "throughput_tok_per_sec": throughput,
            "completion_tokens": completion_tokens,
            "is_streaming": True,
            "mean_itl_ms": itl_stats["mean"],
            "median_itl_ms": itl_stats["median"],
            "p95_itl_ms": itl_stats["p95"],
            "energy_joules": energy_joules,
            "power_watts": power_watts,
            "energy_method": energy_method,
            "energy_vendor": energy_vendor,
        }
        if usage is not None:
            event_data["usage"] = usage_data

        self._bus.publish(EventType.INFERENCE_END, event_data)
        self._bus.publish(EventType.TELEMETRY_RECORD, {"record": record})

    async def stream(
        self,
        messages: Sequence[Message],
        *,
        model: str = "",
        temperature: float = 0.7,
        max_tokens: int = 1024,
        **kwargs: Any,
    ) -> Any:
        """Stream with per-token timing and full telemetry recording."""
        self._bus.publish(
            EventType.INFERENCE_START,
            {
                "model": model,
                "message_count": len(messages),
            },
        )

        t0 = time.time()
        token_timestamps: list[float] = []
        token_count = 0

        energy_sample: Optional[Any] = None
        gpu_sample: Optional[GpuSample] = None

        if self._energy_monitor is not None:
            with self._energy_monitor.sample() as energy_sample:
                async for token in self._inner.stream(
                    messages,
                    model=model,
                    temperature=temperature,
                    max_tokens=max_tokens,
                    **kwargs,
                ):
                    token_timestamps.append(time.time())
                    token_count += 1
                    yield token
        elif self._gpu_monitor is not None:
            with self._gpu_monitor.sample() as gpu_sample:
                async for token in self._inner.stream(
                    messages,
                    model=model,
                    temperature=temperature,
                    max_tokens=max_tokens,
                    **kwargs,
                ):
                    token_timestamps.append(time.time())
                    token_count += 1
                    yield token
        else:
            async for token in self._inner.stream(
                messages,
                model=model,
                temperature=temperature,
                max_tokens=max_tokens,
                **kwargs,
            ):
                token_timestamps.append(time.time())
                token_count += 1
                yield token

        self._publish_stream_telemetry(
            model=model,
            t0=t0,
            token_timestamps=token_timestamps,
            fallback_token_count=token_count,
            energy_sample=energy_sample,
            gpu_sample=gpu_sample,
        )

    async def stream_full(
        self,
        messages: Sequence[Message],
        *,
        model: str,
        temperature: float = 0.7,
        max_tokens: int = 1024,
        **kwargs: Any,
    ) -> AsyncIterator["StreamChunk"]:
        """Stream rich chunks with timing, usage, and energy telemetry.

        Telemetry is published only after the inner stream completes
        successfully.
        """
        self._bus.publish(
            EventType.INFERENCE_START,
            {
                "model": model,
                "message_count": len(messages),
            },
        )

        t0 = time.time()
        token_timestamps: list[float] = []
        content_chunk_count = 0
        usage: Optional[Dict[str, Any]] = None
        energy_sample: Optional[Any] = None
        gpu_sample: Optional[GpuSample] = None

        if self._energy_monitor is not None:
            with self._energy_monitor.sample() as energy_sample:
                async for chunk in self._inner.stream_full(
                    messages,
                    model=model,
                    temperature=temperature,
                    max_tokens=max_tokens,
                    **kwargs,
                ):
                    if chunk.content:
                        token_timestamps.append(time.time())
                        content_chunk_count += 1
                    if chunk.usage is not None:
                        usage = chunk.usage
                    yield chunk
        elif self._gpu_monitor is not None:
            with self._gpu_monitor.sample() as gpu_sample:
                async for chunk in self._inner.stream_full(
                    messages,
                    model=model,
                    temperature=temperature,
                    max_tokens=max_tokens,
                    **kwargs,
                ):
                    if chunk.content:
                        token_timestamps.append(time.time())
                        content_chunk_count += 1
                    if chunk.usage is not None:
                        usage = chunk.usage
                    yield chunk
        else:
            async for chunk in self._inner.stream_full(
                messages,
                model=model,
                temperature=temperature,
                max_tokens=max_tokens,
                **kwargs,
            ):
                if chunk.content:
                    token_timestamps.append(time.time())
                    content_chunk_count += 1
                if chunk.usage is not None:
                    usage = chunk.usage
                yield chunk

        self._publish_stream_telemetry(
            model=model,
            t0=t0,
            token_timestamps=token_timestamps,
            fallback_token_count=content_chunk_count,
            energy_sample=energy_sample,
            gpu_sample=gpu_sample,
            usage=usage,
        )

    def list_models(self) -> List[str]:
        return self._inner.list_models()

    def health(self) -> bool:
        return self._inner.health()

    def close(self) -> None:
        self._inner.close()


__all__ = ["InstrumentedEngine", "_compute_itl_stats", "_percentile"]