Skip to content

Personal AI, On Personal Devices

Python

feat: KV-cache-aware FLOPs/energy with full-count dollar cost

Jon Saad-Falcon26a4e2e65eb7
diff --git a/src/openjarvis/core/types.py b/src/openjarvis/core/types.py
index a8629752..bfde1a41 100644
--- a/src/openjarvis/core/types.py
+++ b/src/openjarvis/core/types.py
@@ -132,6 +132,7 @@ class TelemetryRecord:
timestamp: float
model_id: str
prompt_tokens: int = 0
+ prompt_tokens_evaluated: int = 0 # KV-cache-aware: actual tokens processed
completion_tokens: int = 0
total_tokens: int = 0
latency_seconds: float = 0.0
diff --git a/src/openjarvis/engine/_openai_compat.py b/src/openjarvis/engine/_openai_compat.py
index bbc3cf00..0eb430f3 100644
--- a/src/openjarvis/engine/_openai_compat.py
+++ b/src/openjarvis/engine/_openai_compat.py
@@ -95,8 +95,9 @@ class _OpenAICompatibleEngine(InferenceEngine):
choice = choices[0]
usage = data.get("usage", {})
# Ensure prompt_tokens reflects the full prompt size (including
- # system prompt and all conversation history) without assuming
- # KV-cache savings.
+ # system prompt and all conversation history).
+ # OpenAI-compat APIs (vLLM, SGLang) report full counts — KV
+ # caching is transparent, so evaluated == full.
reported_prompt = usage.get("prompt_tokens", 0)
estimated_prompt = estimate_prompt_tokens(messages)
prompt_tokens = max(reported_prompt, estimated_prompt)
@@ -105,6 +106,7 @@ class _OpenAICompatibleEngine(InferenceEngine):
"content": choice["message"].get("content") or "",
"usage": {
"prompt_tokens": prompt_tokens,
+ "prompt_tokens_evaluated": reported_prompt or prompt_tokens,
"completion_tokens": completion_tokens,
"total_tokens": prompt_tokens + completion_tokens,
},
diff --git a/src/openjarvis/engine/ollama.py b/src/openjarvis/engine/ollama.py
index 1b219368..a757293e 100644
--- a/src/openjarvis/engine/ollama.py
+++ b/src/openjarvis/engine/ollama.py
@@ -106,19 +106,24 @@ class OllamaEngine(InferenceEngine):
f"Ollama returned {exc.response.status_code}: {body}"
) from exc
data = resp.json()
- # Ollama's prompt_eval_count may exclude KV-cached tokens
- # (system prompt, prior turns). Use the larger of the reported
- # count and a cache-agnostic estimate so that cost / FLOPs /
- # energy calculations reflect the full prompt size.
+ # prompt_eval_count = tokens actually evaluated (KV-cache-aware).
+ # estimate_prompt_tokens = full prompt size (for cost comparison).
+ # We report both so downstream can use the right one:
+ # prompt_tokens → full size (what cloud would charge)
+ # prompt_tokens_evaluated → actual compute (with KV cache)
reported_prompt = data.get("prompt_eval_count", 0)
estimated_prompt = estimate_prompt_tokens(messages)
prompt_tokens = max(reported_prompt, estimated_prompt)
+ prompt_tokens_evaluated = (
+ reported_prompt if reported_prompt > 0 else prompt_tokens
+ )
completion_tokens = data.get("eval_count", 0)
content = data.get("message", {}).get("content", "")
result: Dict[str, Any] = {
"content": content,
"usage": {
"prompt_tokens": prompt_tokens,
+ "prompt_tokens_evaluated": prompt_tokens_evaluated,
"completion_tokens": completion_tokens,
"total_tokens": prompt_tokens + completion_tokens,
},
@@ -183,16 +188,17 @@ class OllamaEngine(InferenceEngine):
if content:
yield content
if chunk.get("done", False):
- # Capture usage from final chunk. Use the
- # cache-agnostic estimate for prompt tokens to
- # avoid under-counting when Ollama's KV cache
- # suppresses prompt_eval_count.
reported_prompt = chunk.get("prompt_eval_count", 0)
est_prompt = estimate_prompt_tokens(messages)
full_prompt = max(reported_prompt, est_prompt)
+ evaluated = (
+ reported_prompt if reported_prompt > 0
+ else full_prompt
+ )
comp = chunk.get("eval_count", 0)
self._last_stream_usage = {
"prompt_tokens": full_prompt,
+ "prompt_tokens_evaluated": evaluated,
"completion_tokens": comp,
"total_tokens": full_prompt + comp,
}
diff --git a/src/openjarvis/server/routes.py b/src/openjarvis/server/routes.py
index 6c163f65..50fb645b 100644
--- a/src/openjarvis/server/routes.py
+++ b/src/openjarvis/server/routes.py
@@ -470,6 +470,9 @@ async def savings(request: Request):
),
total_calls=sum(m.call_count for m in local_models),
session_start=session_start if session_start else 0.0,
+ prompt_tokens_evaluated=sum(
+ m.prompt_tokens_evaluated for m in local_models
+ ),
)
return savings_to_dict(result)
finally:
diff --git a/src/openjarvis/server/savings.py b/src/openjarvis/server/savings.py
index 0c6fbc0d..ae807139 100644
--- a/src/openjarvis/server/savings.py
+++ b/src/openjarvis/server/savings.py
@@ -89,20 +89,28 @@ def compute_savings(
completion_tokens: int,
total_calls: int = 0,
session_start: float = 0.0,
+ prompt_tokens_evaluated: int = 0,
) -> SavingsSummary:
"""Compute savings vs cloud providers given token counts.
- Token counting assumptions (these match cloud provider billing):
- - **System prompt included**: ``prompt_tokens`` includes system
- prompt tokens for every API call, matching what cloud providers
- charge.
- - **No KV-cache discount**: Each call's ``prompt_tokens`` reflects
- the full context size (system prompt + conversation history).
- Even though local engines may use KV caching internally, the
- savings comparison uses the uncached token count because cloud
- providers bill for all input tokens on every request.
+ Two token counts are used:
+
+ - ``prompt_tokens`` — full prompt size (system prompt + all history).
+ Used for **dollar cost** comparison since cloud providers bill for
+ every input token on every request.
+ - ``prompt_tokens_evaluated`` — actual tokens processed (KV-cache-
+ aware). In multi-turn conversations, subsequent turns only
+ evaluate new tokens; the system prompt and prior context are
+ served from KV cache. Used for **FLOPs** and **energy**
+ calculations since these reflect actual compute.
+
+ If ``prompt_tokens_evaluated`` is 0 (e.g. old telemetry without the
+ column), it falls back to ``prompt_tokens``.
"""
+ if prompt_tokens_evaluated <= 0:
+ prompt_tokens_evaluated = prompt_tokens
total_tokens = prompt_tokens + completion_tokens
+ total_tokens_evaluated = prompt_tokens_evaluated + completion_tokens
providers: List[ProviderSavings] = []
now = time.time()
@@ -116,10 +124,15 @@ def compute_savings(
output_cost = (completion_tokens / 1_000_000) * pricing["output_per_1m"]
total_cost = input_cost + output_cost
- # No-KV-cache FLOPs: P * N * (N+1)
+ # KV-cache-aware FLOPs: 2 * P * T_evaluated
+ # Only the actually-evaluated tokens require compute; cached
+ # tokens from prior turns are served from KV cache.
params_b = pricing.get("params_b", 200.0)
params = params_b * 1e9
- flops = params * total_tokens * (total_tokens + 1) if total_tokens > 0 else 0.0
+ flops = (
+ 2.0 * params * total_tokens_evaluated
+ if total_tokens_evaluated > 0 else 0.0
+ )
# Derive Wh-per-FLOP from the provider's per-token constants:
# energy_wh_per_1k_tokens / (1000 * flops_per_token) = Wh per FLOP
wh_per_flop = pricing["energy_wh_per_1k_tokens"] / (
diff --git a/src/openjarvis/telemetry/aggregator.py b/src/openjarvis/telemetry/aggregator.py
index c055a51f..31754888 100644
--- a/src/openjarvis/telemetry/aggregator.py
+++ b/src/openjarvis/telemetry/aggregator.py
@@ -19,6 +19,7 @@ class ModelStats:
call_count: int = 0
total_tokens: int = 0
prompt_tokens: int = 0
+ prompt_tokens_evaluated: int = 0
completion_tokens: int = 0
total_latency: float = 0.0
avg_latency: float = 0.0
@@ -128,11 +129,17 @@ class TelemetryAggregator:
# Build optional columns for new fields (graceful on old DBs)
extra_cols = ""
+ has_pte = self._safe_col("prompt_tokens_evaluated")
has_tpj = self._safe_col("tokens_per_joule")
has_derived = self._safe_col("energy_per_output_token_joules")
has_phase = self._safe_col("prefill_energy_joules")
has_itl = self._safe_col("mean_itl_ms")
+ if has_pte:
+ extra_cols += (
+ ", SUM(prompt_tokens_evaluated)"
+ " AS prompt_tokens_evaluated"
+ )
if has_tpj:
extra_cols += (
", AVG(tokens_per_joule) AS avg_tokens_per_joule"
@@ -189,6 +196,10 @@ class TelemetryAggregator:
avg_gpu_utilization_pct=r["avg_gpu_utilization_pct"] or 0.0,
avg_throughput_tok_per_sec=r["avg_throughput_tok_per_sec"] or 0.0,
)
+ if has_pte:
+ ms.prompt_tokens_evaluated = (
+ r["prompt_tokens_evaluated"] or 0
+ )
if has_tpj:
ms.avg_tokens_per_joule = r["avg_tokens_per_joule"] or 0.0
if has_derived:
diff --git a/src/openjarvis/telemetry/instrumented_engine.py b/src/openjarvis/telemetry/instrumented_engine.py
index e8934681..1ebe5d06 100644
--- a/src/openjarvis/telemetry/instrumented_engine.py
+++ b/src/openjarvis/telemetry/instrumented_engine.py
@@ -109,6 +109,7 @@ class InstrumentedEngine(InferenceEngine):
usage = result.get("usage", {})
completion_tokens = usage.get("completion_tokens", 0)
+ prompt_tokens_evaluated = usage.get("prompt_tokens_evaluated", 0)
ttft = result.get("ttft", 0.0)
throughput = completion_tokens / latency if latency > 0 else 0.0
@@ -186,6 +187,7 @@ class InstrumentedEngine(InferenceEngine):
timestamp=t0,
model_id=model,
prompt_tokens=prompt_tok,
+ prompt_tokens_evaluated=prompt_tokens_evaluated or prompt_tok,
completion_tokens=completion_tokens,
total_tokens=prompt_tok + completion_tokens,
latency_seconds=latency,
diff --git a/src/openjarvis/telemetry/store.py b/src/openjarvis/telemetry/store.py
index 8b4702c7..eea89b84 100644
--- a/src/openjarvis/telemetry/store.py
+++ b/src/openjarvis/telemetry/store.py
@@ -20,6 +20,7 @@ CREATE TABLE IF NOT EXISTS telemetry (
engine TEXT NOT NULL DEFAULT '',
agent TEXT NOT NULL DEFAULT '',
prompt_tokens INTEGER NOT NULL DEFAULT 0,
+ prompt_tokens_evaluated INTEGER NOT NULL DEFAULT 0,
completion_tokens INTEGER NOT NULL DEFAULT 0,
total_tokens INTEGER NOT NULL DEFAULT 0,
latency_seconds REAL NOT NULL DEFAULT 0.0,
@@ -59,7 +60,7 @@ CREATE TABLE IF NOT EXISTS telemetry (
_INSERT = """\
INSERT INTO telemetry (
timestamp, model_id, engine, agent,
- prompt_tokens, completion_tokens, total_tokens,
+ prompt_tokens, prompt_tokens_evaluated, completion_tokens, total_tokens,
latency_seconds, ttft, cost_usd, energy_joules, power_watts,
gpu_utilization_pct, gpu_memory_used_gb, gpu_temperature_c,
throughput_tok_per_sec, prefill_latency_seconds, decode_latency_seconds,
@@ -75,7 +76,7 @@ INSERT INTO telemetry (
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
- ?, ?, ?, ?, ?, ?, ?, ?
+ ?, ?, ?, ?, ?, ?, ?, ?, ?
)
"""
@@ -105,6 +106,7 @@ _MIGRATE_COLUMNS = [
("p99_itl_ms", "REAL NOT NULL DEFAULT 0.0"),
("std_itl_ms", "REAL NOT NULL DEFAULT 0.0"),
("is_streaming", "INTEGER NOT NULL DEFAULT 0"),
+ ("prompt_tokens_evaluated", "INTEGER NOT NULL DEFAULT 0"),
]
@@ -139,6 +141,7 @@ class TelemetryStore:
rec.engine,
rec.agent,
rec.prompt_tokens,
+ rec.prompt_tokens_evaluated,
rec.completion_tokens,
rec.total_tokens,
rec.latency_seconds,