Personal AI, On Personal Devices
Python
feat: KV-cache-aware FLOPs/energy with full-count dollar cost
Jon Saad-Falcon26a4e2e65eb7
diff --git a/src/openjarvis/core/types.py b/src/openjarvis/core/types.pyindex a8629752..bfde1a41 100644--- a/src/openjarvis/core/types.py+++ b/src/openjarvis/core/types.py@@ -132,6 +132,7 @@ class TelemetryRecord:timestamp: floatmodel_id: strprompt_tokens: int = 0+ prompt_tokens_evaluated: int = 0 # KV-cache-aware: actual tokens processedcompletion_tokens: int = 0total_tokens: int = 0latency_seconds: float = 0.0diff --git a/src/openjarvis/engine/_openai_compat.py b/src/openjarvis/engine/_openai_compat.pyindex bbc3cf00..0eb430f3 100644--- a/src/openjarvis/engine/_openai_compat.py+++ b/src/openjarvis/engine/_openai_compat.py@@ -95,8 +95,9 @@ class _OpenAICompatibleEngine(InferenceEngine):choice = choices[0]usage = data.get("usage", {})# Ensure prompt_tokens reflects the full prompt size (including- # system prompt and all conversation history) without assuming- # KV-cache savings.+ # system prompt and all conversation history).+ # OpenAI-compat APIs (vLLM, SGLang) report full counts — KV+ # caching is transparent, so evaluated == full.reported_prompt = usage.get("prompt_tokens", 0)estimated_prompt = estimate_prompt_tokens(messages)prompt_tokens = max(reported_prompt, estimated_prompt)@@ -105,6 +106,7 @@ class _OpenAICompatibleEngine(InferenceEngine):"content": choice["message"].get("content") or "","usage": {"prompt_tokens": prompt_tokens,+ "prompt_tokens_evaluated": reported_prompt or prompt_tokens,"completion_tokens": completion_tokens,"total_tokens": prompt_tokens + completion_tokens,},diff --git a/src/openjarvis/engine/ollama.py b/src/openjarvis/engine/ollama.pyindex 1b219368..a757293e 100644--- a/src/openjarvis/engine/ollama.py+++ b/src/openjarvis/engine/ollama.py@@ -106,19 +106,24 @@ class OllamaEngine(InferenceEngine):f"Ollama returned {exc.response.status_code}: {body}") from excdata = resp.json()- # Ollama's prompt_eval_count may exclude KV-cached tokens- # (system prompt, prior turns). Use the larger of the reported- # count and a cache-agnostic estimate so that cost / FLOPs /- # energy calculations reflect the full prompt size.+ # prompt_eval_count = tokens actually evaluated (KV-cache-aware).+ # estimate_prompt_tokens = full prompt size (for cost comparison).+ # We report both so downstream can use the right one:+ # prompt_tokens → full size (what cloud would charge)+ # prompt_tokens_evaluated → actual compute (with KV cache)reported_prompt = data.get("prompt_eval_count", 0)estimated_prompt = estimate_prompt_tokens(messages)prompt_tokens = max(reported_prompt, estimated_prompt)+ prompt_tokens_evaluated = (+ reported_prompt if reported_prompt > 0 else prompt_tokens+ )completion_tokens = data.get("eval_count", 0)content = data.get("message", {}).get("content", "")result: Dict[str, Any] = {"content": content,"usage": {"prompt_tokens": prompt_tokens,+ "prompt_tokens_evaluated": prompt_tokens_evaluated,"completion_tokens": completion_tokens,"total_tokens": prompt_tokens + completion_tokens,},@@ -183,16 +188,17 @@ class OllamaEngine(InferenceEngine):if content:yield contentif chunk.get("done", False):- # Capture usage from final chunk. Use the- # cache-agnostic estimate for prompt tokens to- # avoid under-counting when Ollama's KV cache- # suppresses prompt_eval_count.reported_prompt = chunk.get("prompt_eval_count", 0)est_prompt = estimate_prompt_tokens(messages)full_prompt = max(reported_prompt, est_prompt)+ evaluated = (+ reported_prompt if reported_prompt > 0+ else full_prompt+ )comp = chunk.get("eval_count", 0)self._last_stream_usage = {"prompt_tokens": full_prompt,+ "prompt_tokens_evaluated": evaluated,"completion_tokens": comp,"total_tokens": full_prompt + comp,}diff --git a/src/openjarvis/server/routes.py b/src/openjarvis/server/routes.pyindex 6c163f65..50fb645b 100644--- a/src/openjarvis/server/routes.py+++ b/src/openjarvis/server/routes.py@@ -470,6 +470,9 @@ async def savings(request: Request):),total_calls=sum(m.call_count for m in local_models),session_start=session_start if session_start else 0.0,+ prompt_tokens_evaluated=sum(+ m.prompt_tokens_evaluated for m in local_models+ ),)return savings_to_dict(result)finally:diff --git a/src/openjarvis/server/savings.py b/src/openjarvis/server/savings.pyindex 0c6fbc0d..ae807139 100644--- a/src/openjarvis/server/savings.py+++ b/src/openjarvis/server/savings.py@@ -89,20 +89,28 @@ def compute_savings(completion_tokens: int,total_calls: int = 0,session_start: float = 0.0,+ prompt_tokens_evaluated: int = 0,) -> SavingsSummary:"""Compute savings vs cloud providers given token counts.- Token counting assumptions (these match cloud provider billing):- - **System prompt included**: ``prompt_tokens`` includes system- prompt tokens for every API call, matching what cloud providers- charge.- - **No KV-cache discount**: Each call's ``prompt_tokens`` reflects- the full context size (system prompt + conversation history).- Even though local engines may use KV caching internally, the- savings comparison uses the uncached token count because cloud- providers bill for all input tokens on every request.+ Two token counts are used:++ - ``prompt_tokens`` — full prompt size (system prompt + all history).+ Used for **dollar cost** comparison since cloud providers bill for+ every input token on every request.+ - ``prompt_tokens_evaluated`` — actual tokens processed (KV-cache-+ aware). In multi-turn conversations, subsequent turns only+ evaluate new tokens; the system prompt and prior context are+ served from KV cache. Used for **FLOPs** and **energy**+ calculations since these reflect actual compute.++ If ``prompt_tokens_evaluated`` is 0 (e.g. old telemetry without the+ column), it falls back to ``prompt_tokens``."""+ if prompt_tokens_evaluated <= 0:+ prompt_tokens_evaluated = prompt_tokenstotal_tokens = prompt_tokens + completion_tokens+ total_tokens_evaluated = prompt_tokens_evaluated + completion_tokensproviders: List[ProviderSavings] = []now = time.time()@@ -116,10 +124,15 @@ def compute_savings(output_cost = (completion_tokens / 1_000_000) * pricing["output_per_1m"]total_cost = input_cost + output_cost- # No-KV-cache FLOPs: P * N * (N+1)+ # KV-cache-aware FLOPs: 2 * P * T_evaluated+ # Only the actually-evaluated tokens require compute; cached+ # tokens from prior turns are served from KV cache.params_b = pricing.get("params_b", 200.0)params = params_b * 1e9- flops = params * total_tokens * (total_tokens + 1) if total_tokens > 0 else 0.0+ flops = (+ 2.0 * params * total_tokens_evaluated+ if total_tokens_evaluated > 0 else 0.0+ )# Derive Wh-per-FLOP from the provider's per-token constants:# energy_wh_per_1k_tokens / (1000 * flops_per_token) = Wh per FLOPwh_per_flop = pricing["energy_wh_per_1k_tokens"] / (diff --git a/src/openjarvis/telemetry/aggregator.py b/src/openjarvis/telemetry/aggregator.pyindex c055a51f..31754888 100644--- a/src/openjarvis/telemetry/aggregator.py+++ b/src/openjarvis/telemetry/aggregator.py@@ -19,6 +19,7 @@ class ModelStats:call_count: int = 0total_tokens: int = 0prompt_tokens: int = 0+ prompt_tokens_evaluated: int = 0completion_tokens: int = 0total_latency: float = 0.0avg_latency: float = 0.0@@ -128,11 +129,17 @@ class TelemetryAggregator:# Build optional columns for new fields (graceful on old DBs)extra_cols = ""+ has_pte = self._safe_col("prompt_tokens_evaluated")has_tpj = self._safe_col("tokens_per_joule")has_derived = self._safe_col("energy_per_output_token_joules")has_phase = self._safe_col("prefill_energy_joules")has_itl = self._safe_col("mean_itl_ms")+ if has_pte:+ extra_cols += (+ ", SUM(prompt_tokens_evaluated)"+ " AS prompt_tokens_evaluated"+ )if has_tpj:extra_cols += (", AVG(tokens_per_joule) AS avg_tokens_per_joule"@@ -189,6 +196,10 @@ class TelemetryAggregator:avg_gpu_utilization_pct=r["avg_gpu_utilization_pct"] or 0.0,avg_throughput_tok_per_sec=r["avg_throughput_tok_per_sec"] or 0.0,)+ if has_pte:+ ms.prompt_tokens_evaluated = (+ r["prompt_tokens_evaluated"] or 0+ )if has_tpj:ms.avg_tokens_per_joule = r["avg_tokens_per_joule"] or 0.0if has_derived:diff --git a/src/openjarvis/telemetry/instrumented_engine.py b/src/openjarvis/telemetry/instrumented_engine.pyindex e8934681..1ebe5d06 100644--- a/src/openjarvis/telemetry/instrumented_engine.py+++ b/src/openjarvis/telemetry/instrumented_engine.py@@ -109,6 +109,7 @@ class InstrumentedEngine(InferenceEngine):usage = result.get("usage", {})completion_tokens = usage.get("completion_tokens", 0)+ prompt_tokens_evaluated = usage.get("prompt_tokens_evaluated", 0)ttft = result.get("ttft", 0.0)throughput = completion_tokens / latency if latency > 0 else 0.0@@ -186,6 +187,7 @@ class InstrumentedEngine(InferenceEngine):timestamp=t0,model_id=model,prompt_tokens=prompt_tok,+ prompt_tokens_evaluated=prompt_tokens_evaluated or prompt_tok,completion_tokens=completion_tokens,total_tokens=prompt_tok + completion_tokens,latency_seconds=latency,diff --git a/src/openjarvis/telemetry/store.py b/src/openjarvis/telemetry/store.pyindex 8b4702c7..eea89b84 100644--- a/src/openjarvis/telemetry/store.py+++ b/src/openjarvis/telemetry/store.py@@ -20,6 +20,7 @@ CREATE TABLE IF NOT EXISTS telemetry (engine TEXT NOT NULL DEFAULT '',agent TEXT NOT NULL DEFAULT '',prompt_tokens INTEGER NOT NULL DEFAULT 0,+ prompt_tokens_evaluated INTEGER NOT NULL DEFAULT 0,completion_tokens INTEGER NOT NULL DEFAULT 0,total_tokens INTEGER NOT NULL DEFAULT 0,latency_seconds REAL NOT NULL DEFAULT 0.0,@@ -59,7 +60,7 @@ CREATE TABLE IF NOT EXISTS telemetry (_INSERT = """\INSERT INTO telemetry (timestamp, model_id, engine, agent,- prompt_tokens, completion_tokens, total_tokens,+ prompt_tokens, prompt_tokens_evaluated, completion_tokens, total_tokens,latency_seconds, ttft, cost_usd, energy_joules, power_watts,gpu_utilization_pct, gpu_memory_used_gb, gpu_temperature_c,throughput_tok_per_sec, prefill_latency_seconds, decode_latency_seconds,@@ -75,7 +76,7 @@ INSERT INTO telemetry (?, ?, ?, ?, ?, ?, ?, ?, ?, ?,?, ?, ?, ?, ?, ?, ?, ?, ?, ?,?, ?, ?, ?, ?, ?, ?, ?, ?, ?,- ?, ?, ?, ?, ?, ?, ?, ?+ ?, ?, ?, ?, ?, ?, ?, ?, ?)"""@@ -105,6 +106,7 @@ _MIGRATE_COLUMNS = [("p99_itl_ms", "REAL NOT NULL DEFAULT 0.0"),("std_itl_ms", "REAL NOT NULL DEFAULT 0.0"),("is_streaming", "INTEGER NOT NULL DEFAULT 0"),+ ("prompt_tokens_evaluated", "INTEGER NOT NULL DEFAULT 0"),]@@ -139,6 +141,7 @@ class TelemetryStore:rec.engine,rec.agent,rec.prompt_tokens,+ rec.prompt_tokens_evaluated,rec.completion_tokens,rec.total_tokens,rec.latency_seconds,