Personal AI, On Personal Devices
Python
fix: count full prompt tokens without KV-cache assumption
Jon Saad-Falcon3777086d3aeb
diff --git a/desktop/src/components/SavingsDashboard.tsx b/desktop/src/components/SavingsDashboard.tsxindex df67050b..3f5937e2 100644--- a/desktop/src/components/SavingsDashboard.tsx+++ b/desktop/src/components/SavingsDashboard.tsx@@ -34,6 +34,7 @@ interface SavingsData {heavy_low: number;heavy_high: number;};+ token_counting_version?: number;}// ---------------------------------------------------------------------------@@ -335,6 +336,7 @@ export function SavingsDashboard({ apiUrl }: { apiUrl: string }) {dollar_savings: dollarSavings,energy_wh_saved: energySaved,flops_saved: flopsSaved,+ token_counting_version: data.token_counting_version ?? 1,},}).catch(() => {});}, [data, optInEnabled, displayName, anonId]);diff --git a/frontend/src/App.tsx b/frontend/src/App.tsxindex 617a034f..f619379a 100644--- a/frontend/src/App.tsx+++ b/frontend/src/App.tsx@@ -88,6 +88,7 @@ export default function App() {dollar_savings: dollarSavings,energy_wh_saved: energySaved,flops_saved: flopsSaved,+ token_counting_version: data.token_counting_version ?? 1,});}})diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.tsindex 0bc2cfe4..201e9c7c 100644--- a/frontend/src/lib/api.ts+++ b/frontend/src/lib/api.ts@@ -589,6 +589,7 @@ export interface SavingsSubmission {dollar_savings: number;energy_wh_saved: number;flops_saved: number;+ token_counting_version?: number;}export async function submitSavings(data: SavingsSubmission): Promise<boolean> {diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.tsindex a01205b3..747e1d25 100644--- a/frontend/src/types/index.ts+++ b/frontend/src/types/index.ts@@ -123,6 +123,7 @@ export interface SavingsData {total_tokens: number;local_cost: number;per_provider: ProviderSavings[];+ token_counting_version?: number;}export interface ServerInfo {diff --git a/src/openjarvis/engine/_base.py b/src/openjarvis/engine/_base.pyindex 4207ba65..3b12f34f 100644--- a/src/openjarvis/engine/_base.py+++ b/src/openjarvis/engine/_base.py@@ -38,4 +38,28 @@ def messages_to_dicts(messages: Sequence[Message]) -> List[Dict[str, Any]]:return out-__all__ = ["EngineConnectionError", "InferenceEngine", "messages_to_dicts"]+def estimate_prompt_tokens(messages: Sequence[Message]) -> int:+ """Estimate full prompt token count from message content.++ Ollama's ``prompt_eval_count`` may report only *newly evaluated*+ tokens when KV-cache hits occur, under-counting the system prompt+ and earlier conversation turns. This helper provides a+ cache-agnostic estimate so that downstream cost / FLOPs / energy+ calculations reflect the true prompt size — matching what a cloud+ provider would charge.++ Uses ~4 characters per token (standard BPE average for English) plus+ a small per-message overhead for role markers and separators.+ """+ total_chars = sum(len(m.content) for m in messages)+ # ~4 tokens overhead per message for role markers / separators+ overhead = len(messages) * 4+ return max(1, total_chars // 4 + overhead)+++__all__ = [+ "EngineConnectionError",+ "InferenceEngine",+ "estimate_prompt_tokens",+ "messages_to_dicts",+]diff --git a/src/openjarvis/engine/_openai_compat.py b/src/openjarvis/engine/_openai_compat.pyindex ba7d3870..bbc3cf00 100644--- a/src/openjarvis/engine/_openai_compat.py+++ b/src/openjarvis/engine/_openai_compat.py@@ -13,6 +13,7 @@ from openjarvis.core.types import Messagefrom openjarvis.engine._base import (EngineConnectionError,InferenceEngine,+ estimate_prompt_tokens,messages_to_dicts,)@@ -93,12 +94,19 @@ class _OpenAICompatibleEngine(InferenceEngine):}choice = choices[0]usage = data.get("usage", {})+ # Ensure prompt_tokens reflects the full prompt size (including+ # system prompt and all conversation history) without assuming+ # KV-cache savings.+ reported_prompt = usage.get("prompt_tokens", 0)+ estimated_prompt = estimate_prompt_tokens(messages)+ prompt_tokens = max(reported_prompt, estimated_prompt)+ completion_tokens = usage.get("completion_tokens", 0)result: Dict[str, Any] = {"content": choice["message"].get("content") or "","usage": {- "prompt_tokens": usage.get("prompt_tokens", 0),- "completion_tokens": usage.get("completion_tokens", 0),- "total_tokens": usage.get("total_tokens", 0),+ "prompt_tokens": prompt_tokens,+ "completion_tokens": completion_tokens,+ "total_tokens": prompt_tokens + completion_tokens,},"model": data.get("model", model),"finish_reason": choice.get("finish_reason", "stop"),diff --git a/src/openjarvis/engine/ollama.py b/src/openjarvis/engine/ollama.pyindex 5e95ef9a..1b219368 100644--- a/src/openjarvis/engine/ollama.py+++ b/src/openjarvis/engine/ollama.py@@ -15,6 +15,7 @@ from openjarvis.core.types import Messagefrom openjarvis.engine._base import (EngineConnectionError,InferenceEngine,+ estimate_prompt_tokens,messages_to_dicts,)@@ -105,7 +106,13 @@ class OllamaEngine(InferenceEngine):f"Ollama returned {exc.response.status_code}: {body}") from excdata = resp.json()- prompt_tokens = data.get("prompt_eval_count", 0)+ # Ollama's prompt_eval_count may exclude KV-cached tokens+ # (system prompt, prior turns). Use the larger of the reported+ # count and a cache-agnostic estimate so that cost / FLOPs /+ # energy calculations reflect the full prompt size.+ reported_prompt = data.get("prompt_eval_count", 0)+ estimated_prompt = estimate_prompt_tokens(messages)+ prompt_tokens = max(reported_prompt, estimated_prompt)completion_tokens = data.get("eval_count", 0)content = data.get("message", {}).get("content", "")result: Dict[str, Any] = {@@ -176,14 +183,18 @@ class OllamaEngine(InferenceEngine):if content:yield contentif chunk.get("done", False):- # Capture usage from final chunk+ # Capture usage from final chunk. Use the+ # cache-agnostic estimate for prompt tokens to+ # avoid under-counting when Ollama's KV cache+ # suppresses prompt_eval_count.+ reported_prompt = chunk.get("prompt_eval_count", 0)+ est_prompt = estimate_prompt_tokens(messages)+ full_prompt = max(reported_prompt, est_prompt)+ comp = chunk.get("eval_count", 0)self._last_stream_usage = {- "prompt_tokens": chunk.get("prompt_eval_count", 0),- "completion_tokens": chunk.get("eval_count", 0),- "total_tokens": (- chunk.get("prompt_eval_count", 0)- + chunk.get("eval_count", 0)- ),+ "prompt_tokens": full_prompt,+ "completion_tokens": comp,+ "total_tokens": full_prompt + comp,}breakexcept (httpx.ConnectError, httpx.TimeoutException) as exc:diff --git a/src/openjarvis/server/routes.py b/src/openjarvis/server/routes.pyindex bf935bef..6c163f65 100644--- a/src/openjarvis/server/routes.py+++ b/src/openjarvis/server/routes.py@@ -476,6 +476,30 @@ async def savings(request: Request):agg.close()[email protected]("/v1/telemetry/reset")+async def reset_telemetry():+ """Clear all stored telemetry records.++ Useful after updating token-counting methodology — clears+ historical records that were computed under the old rules so+ that the savings dashboard and leaderboard submissions start+ fresh with corrected values.+ """+ from openjarvis.core.config import DEFAULT_CONFIG_DIR+ from openjarvis.telemetry.aggregator import TelemetryAggregator++ db_path = DEFAULT_CONFIG_DIR / "telemetry.db"+ if not db_path.exists():+ return {"status": "ok", "records_cleared": 0}++ agg = TelemetryAggregator(db_path)+ try:+ count = agg.clear()+ finally:+ agg.close()+ return {"status": "ok", "records_cleared": count}++@router.get("/v1/info")async def server_info(request: Request):"""Return server configuration: model, agent, engine."""diff --git a/src/openjarvis/server/savings.py b/src/openjarvis/server/savings.pyindex 993af063..0c6fbc0d 100644--- a/src/openjarvis/server/savings.py+++ b/src/openjarvis/server/savings.py@@ -10,6 +10,12 @@ import timefrom dataclasses import asdict, dataclass, fieldfrom typing import Any, Dict, List+# Bump when token-counting methodology changes so that the leaderboard+# can distinguish entries computed under different rules.+# v1 = original (Ollama prompt_eval_count, may under-count due to KV cache)+# v2 = full prompt token count, no KV-cache assumption, system prompt always counted+TOKEN_COUNTING_VERSION: int = 2+# ---------------------------------------------------------------------------# Cloud provider pricing (USD per 1M tokens)# params_b: model size in billions, for no-KV-cache FLOPs@@ -75,6 +81,7 @@ class SavingsSummary:session_duration_hours: float = 0.0avg_cost_per_query: Dict[str, float] = field(default_factory=dict)cloud_agent_equivalent: Dict[str, int] = field(default_factory=dict)+ token_counting_version: int = TOKEN_COUNTING_VERSIONdef compute_savings(@@ -83,7 +90,18 @@ def compute_savings(total_calls: int = 0,session_start: float = 0.0,) -> SavingsSummary:- """Compute savings vs cloud providers given token counts."""+ """Compute savings vs cloud providers given token counts.++ Token counting assumptions (these match cloud provider billing):+ - **System prompt included**: ``prompt_tokens`` includes system+ prompt tokens for every API call, matching what cloud providers+ charge.+ - **No KV-cache discount**: Each call's ``prompt_tokens`` reflects+ the full context size (system prompt + conversation history).+ Even though local engines may use KV caching internally, the+ savings comparison uses the uncached token count because cloud+ providers bill for all input tokens on every request.+ """total_tokens = prompt_tokens + completion_tokensproviders: List[ProviderSavings] = []diff --git a/src/openjarvis/telemetry/flops.py b/src/openjarvis/telemetry/flops.pyindex a9527dcb..67e7b06b 100644--- a/src/openjarvis/telemetry/flops.py+++ b/src/openjarvis/telemetry/flops.py@@ -49,6 +49,10 @@ def estimate_flops(Uses the 2 * P * T approximation where P = params, T = total tokens.Returns (total_flops, flops_per_token).++ ``input_tokens`` must include system-prompt tokens and must *not*+ be reduced for KV-cache reuse — it should represent the full prompt+ size that was sent to the engine."""params_b = _get_params_b(model)total_tokens = input_tokens + output_tokens