Skip to content

Personal AI, On Personal Devices

Python

fix: count full prompt tokens without KV-cache assumption

Jon Saad-Falcon3777086d3aeb
diff --git a/desktop/src/components/SavingsDashboard.tsx b/desktop/src/components/SavingsDashboard.tsx
index df67050b..3f5937e2 100644
--- a/desktop/src/components/SavingsDashboard.tsx
+++ b/desktop/src/components/SavingsDashboard.tsx
@@ -34,6 +34,7 @@ interface SavingsData {
heavy_low: number;
heavy_high: number;
};
+ token_counting_version?: number;
}
// ---------------------------------------------------------------------------
@@ -335,6 +336,7 @@ export function SavingsDashboard({ apiUrl }: { apiUrl: string }) {
dollar_savings: dollarSavings,
energy_wh_saved: energySaved,
flops_saved: flopsSaved,
+ token_counting_version: data.token_counting_version ?? 1,
},
}).catch(() => {});
}, [data, optInEnabled, displayName, anonId]);
diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx
index 617a034f..f619379a 100644
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -88,6 +88,7 @@ export default function App() {
dollar_savings: dollarSavings,
energy_wh_saved: energySaved,
flops_saved: flopsSaved,
+ token_counting_version: data.token_counting_version ?? 1,
});
}
})
diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts
index 0bc2cfe4..201e9c7c 100644
--- a/frontend/src/lib/api.ts
+++ b/frontend/src/lib/api.ts
@@ -589,6 +589,7 @@ export interface SavingsSubmission {
dollar_savings: number;
energy_wh_saved: number;
flops_saved: number;
+ token_counting_version?: number;
}
export async function submitSavings(data: SavingsSubmission): Promise<boolean> {
diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts
index a01205b3..747e1d25 100644
--- a/frontend/src/types/index.ts
+++ b/frontend/src/types/index.ts
@@ -123,6 +123,7 @@ export interface SavingsData {
total_tokens: number;
local_cost: number;
per_provider: ProviderSavings[];
+ token_counting_version?: number;
}
export interface ServerInfo {
diff --git a/src/openjarvis/engine/_base.py b/src/openjarvis/engine/_base.py
index 4207ba65..3b12f34f 100644
--- a/src/openjarvis/engine/_base.py
+++ b/src/openjarvis/engine/_base.py
@@ -38,4 +38,28 @@ def messages_to_dicts(messages: Sequence[Message]) -> List[Dict[str, Any]]:
return out
-__all__ = ["EngineConnectionError", "InferenceEngine", "messages_to_dicts"]
+def estimate_prompt_tokens(messages: Sequence[Message]) -> int:
+ """Estimate full prompt token count from message content.
+
+ Ollama's ``prompt_eval_count`` may report only *newly evaluated*
+ tokens when KV-cache hits occur, under-counting the system prompt
+ and earlier conversation turns. This helper provides a
+ cache-agnostic estimate so that downstream cost / FLOPs / energy
+ calculations reflect the true prompt size — matching what a cloud
+ provider would charge.
+
+ Uses ~4 characters per token (standard BPE average for English) plus
+ a small per-message overhead for role markers and separators.
+ """
+ total_chars = sum(len(m.content) for m in messages)
+ # ~4 tokens overhead per message for role markers / separators
+ overhead = len(messages) * 4
+ return max(1, total_chars // 4 + overhead)
+
+
+__all__ = [
+ "EngineConnectionError",
+ "InferenceEngine",
+ "estimate_prompt_tokens",
+ "messages_to_dicts",
+]
diff --git a/src/openjarvis/engine/_openai_compat.py b/src/openjarvis/engine/_openai_compat.py
index ba7d3870..bbc3cf00 100644
--- a/src/openjarvis/engine/_openai_compat.py
+++ b/src/openjarvis/engine/_openai_compat.py
@@ -13,6 +13,7 @@ from openjarvis.core.types import Message
from openjarvis.engine._base import (
EngineConnectionError,
InferenceEngine,
+ estimate_prompt_tokens,
messages_to_dicts,
)
@@ -93,12 +94,19 @@ class _OpenAICompatibleEngine(InferenceEngine):
}
choice = choices[0]
usage = data.get("usage", {})
+ # Ensure prompt_tokens reflects the full prompt size (including
+ # system prompt and all conversation history) without assuming
+ # KV-cache savings.
+ reported_prompt = usage.get("prompt_tokens", 0)
+ estimated_prompt = estimate_prompt_tokens(messages)
+ prompt_tokens = max(reported_prompt, estimated_prompt)
+ completion_tokens = usage.get("completion_tokens", 0)
result: Dict[str, Any] = {
"content": choice["message"].get("content") or "",
"usage": {
- "prompt_tokens": usage.get("prompt_tokens", 0),
- "completion_tokens": usage.get("completion_tokens", 0),
- "total_tokens": usage.get("total_tokens", 0),
+ "prompt_tokens": prompt_tokens,
+ "completion_tokens": completion_tokens,
+ "total_tokens": prompt_tokens + completion_tokens,
},
"model": data.get("model", model),
"finish_reason": choice.get("finish_reason", "stop"),
diff --git a/src/openjarvis/engine/ollama.py b/src/openjarvis/engine/ollama.py
index 5e95ef9a..1b219368 100644
--- a/src/openjarvis/engine/ollama.py
+++ b/src/openjarvis/engine/ollama.py
@@ -15,6 +15,7 @@ from openjarvis.core.types import Message
from openjarvis.engine._base import (
EngineConnectionError,
InferenceEngine,
+ estimate_prompt_tokens,
messages_to_dicts,
)
@@ -105,7 +106,13 @@ class OllamaEngine(InferenceEngine):
f"Ollama returned {exc.response.status_code}: {body}"
) from exc
data = resp.json()
- prompt_tokens = data.get("prompt_eval_count", 0)
+ # Ollama's prompt_eval_count may exclude KV-cached tokens
+ # (system prompt, prior turns). Use the larger of the reported
+ # count and a cache-agnostic estimate so that cost / FLOPs /
+ # energy calculations reflect the full prompt size.
+ reported_prompt = data.get("prompt_eval_count", 0)
+ estimated_prompt = estimate_prompt_tokens(messages)
+ prompt_tokens = max(reported_prompt, estimated_prompt)
completion_tokens = data.get("eval_count", 0)
content = data.get("message", {}).get("content", "")
result: Dict[str, Any] = {
@@ -176,14 +183,18 @@ class OllamaEngine(InferenceEngine):
if content:
yield content
if chunk.get("done", False):
- # Capture usage from final chunk
+ # Capture usage from final chunk. Use the
+ # cache-agnostic estimate for prompt tokens to
+ # avoid under-counting when Ollama's KV cache
+ # suppresses prompt_eval_count.
+ reported_prompt = chunk.get("prompt_eval_count", 0)
+ est_prompt = estimate_prompt_tokens(messages)
+ full_prompt = max(reported_prompt, est_prompt)
+ comp = chunk.get("eval_count", 0)
self._last_stream_usage = {
- "prompt_tokens": chunk.get("prompt_eval_count", 0),
- "completion_tokens": chunk.get("eval_count", 0),
- "total_tokens": (
- chunk.get("prompt_eval_count", 0)
- + chunk.get("eval_count", 0)
- ),
+ "prompt_tokens": full_prompt,
+ "completion_tokens": comp,
+ "total_tokens": full_prompt + comp,
}
break
except (httpx.ConnectError, httpx.TimeoutException) as exc:
diff --git a/src/openjarvis/server/routes.py b/src/openjarvis/server/routes.py
index bf935bef..6c163f65 100644
--- a/src/openjarvis/server/routes.py
+++ b/src/openjarvis/server/routes.py
@@ -476,6 +476,30 @@ async def savings(request: Request):
agg.close()
[email protected]("/v1/telemetry/reset")
+async def reset_telemetry():
+ """Clear all stored telemetry records.
+
+ Useful after updating token-counting methodology — clears
+ historical records that were computed under the old rules so
+ that the savings dashboard and leaderboard submissions start
+ fresh with corrected values.
+ """
+ from openjarvis.core.config import DEFAULT_CONFIG_DIR
+ from openjarvis.telemetry.aggregator import TelemetryAggregator
+
+ db_path = DEFAULT_CONFIG_DIR / "telemetry.db"
+ if not db_path.exists():
+ return {"status": "ok", "records_cleared": 0}
+
+ agg = TelemetryAggregator(db_path)
+ try:
+ count = agg.clear()
+ finally:
+ agg.close()
+ return {"status": "ok", "records_cleared": count}
+
+
@router.get("/v1/info")
async def server_info(request: Request):
"""Return server configuration: model, agent, engine."""
diff --git a/src/openjarvis/server/savings.py b/src/openjarvis/server/savings.py
index 993af063..0c6fbc0d 100644
--- a/src/openjarvis/server/savings.py
+++ b/src/openjarvis/server/savings.py
@@ -10,6 +10,12 @@ import time
from dataclasses import asdict, dataclass, field
from typing import Any, Dict, List
+# Bump when token-counting methodology changes so that the leaderboard
+# can distinguish entries computed under different rules.
+# v1 = original (Ollama prompt_eval_count, may under-count due to KV cache)
+# v2 = full prompt token count, no KV-cache assumption, system prompt always counted
+TOKEN_COUNTING_VERSION: int = 2
+
# ---------------------------------------------------------------------------
# Cloud provider pricing (USD per 1M tokens)
# params_b: model size in billions, for no-KV-cache FLOPs
@@ -75,6 +81,7 @@ class SavingsSummary:
session_duration_hours: float = 0.0
avg_cost_per_query: Dict[str, float] = field(default_factory=dict)
cloud_agent_equivalent: Dict[str, int] = field(default_factory=dict)
+ token_counting_version: int = TOKEN_COUNTING_VERSION
def compute_savings(
@@ -83,7 +90,18 @@ def compute_savings(
total_calls: int = 0,
session_start: float = 0.0,
) -> SavingsSummary:
- """Compute savings vs cloud providers given token counts."""
+ """Compute savings vs cloud providers given token counts.
+
+ Token counting assumptions (these match cloud provider billing):
+ - **System prompt included**: ``prompt_tokens`` includes system
+ prompt tokens for every API call, matching what cloud providers
+ charge.
+ - **No KV-cache discount**: Each call's ``prompt_tokens`` reflects
+ the full context size (system prompt + conversation history).
+ Even though local engines may use KV caching internally, the
+ savings comparison uses the uncached token count because cloud
+ providers bill for all input tokens on every request.
+ """
total_tokens = prompt_tokens + completion_tokens
providers: List[ProviderSavings] = []
diff --git a/src/openjarvis/telemetry/flops.py b/src/openjarvis/telemetry/flops.py
index a9527dcb..67e7b06b 100644
--- a/src/openjarvis/telemetry/flops.py
+++ b/src/openjarvis/telemetry/flops.py
@@ -49,6 +49,10 @@ def estimate_flops(
Uses the 2 * P * T approximation where P = params, T = total tokens.
Returns (total_flops, flops_per_token).
+
+ ``input_tokens`` must include system-prompt tokens and must *not*
+ be reduced for KV-cache reuse — it should represent the full prompt
+ size that was sent to the engine.
"""
params_b = _get_params_b(model)
total_tokens = input_tokens + output_tokens