Skip to content

Speakr is a personal, self-hosted web application designed for transcribing audio recordings

Python

Track and display prompt-cache token usage (cache reads/writes) in admin token stats

Murtaza Nasirc884f16238ab
diff --git a/src/api/admin.py b/src/api/admin.py
index bf75c796..00a97ac3 100644
--- a/src/api/admin.py
+++ b/src/api/admin.py
@@ -497,6 +497,8 @@ def admin_get_token_stats():
func.sum(TokenUsage.total_tokens).label('tokens'),
func.sum(TokenUsage.cost).label('cost'),
func.sum(TokenUsage.request_count).label('requests'),
+ func.sum(TokenUsage.cached_tokens).label('cached_tokens'),
+ func.sum(TokenUsage.cache_write_tokens).label('cache_write_tokens'),
).filter(
extract('year', TokenUsage.date) == today.year,
extract('month', TokenUsage.date) == today.month,
@@ -507,6 +509,8 @@ def admin_get_token_stats():
'tokens': int(r.tokens or 0),
'cost': float(r.cost or 0.0),
'requests': int(r.requests or 0),
+ 'cached_tokens': int(r.cached_tokens or 0),
+ 'cache_write_tokens': int(r.cache_write_tokens or 0),
'is_embedding': token_tracker.is_embedding_op(r.operation_type),
}
for r in op_rows
@@ -521,6 +525,8 @@ def admin_get_token_stats():
'llm_cost': current_month.get('llm_cost', 0),
'embedding_tokens': current_month.get('embedding_tokens', 0),
'embedding_cost': current_month.get('embedding_cost', 0),
+ 'cached_tokens': current_month.get('cached_tokens', 0),
+ 'cache_write_tokens': current_month.get('cache_write_tokens', 0),
'by_operation': by_operation,
},
'user_count_with_usage': len([u for u in user_stats if u['current_usage'] > 0]),
diff --git a/src/init_db.py b/src/init_db.py
index ce4e5260..0333d4ff 100644
--- a/src/init_db.py
+++ b/src/init_db.py
@@ -233,6 +233,13 @@ def initialize_database(app):
if add_column_if_not_exists(engine, 'initial_prompt_template', 'hotwords', 'TEXT'):
app.logger.info("Added hotwords column to initial_prompt_template table")
+ # Prompt-cache token tracking: cached_tokens (prefix-cache reads) and
+ # cache_write_tokens (tokens written to the cache) per usage row.
+ if add_column_if_not_exists(engine, 'token_usage', 'cached_tokens', 'INTEGER DEFAULT 0'):
+ app.logger.info("Added cached_tokens column to token_usage table")
+ if add_column_if_not_exists(engine, 'token_usage', 'cache_write_tokens', 'INTEGER DEFAULT 0'):
+ app.logger.info("Added cache_write_tokens column to token_usage table")
+
# Add speaker voice profile embedding fields
if add_column_if_not_exists(engine, 'speaker', 'average_embedding', 'BLOB'):
app.logger.info("Added average_embedding column to speaker table")
diff --git a/src/models/token_usage.py b/src/models/token_usage.py
index cd7461fd..59e2be1e 100644
--- a/src/models/token_usage.py
+++ b/src/models/token_usage.py
@@ -20,6 +20,13 @@ class TokenUsage(db.Model):
completion_tokens = db.Column(db.Integer, default=0)
total_tokens = db.Column(db.Integer, default=0)
+ # Prompt-cache counts (subset of prompt_tokens). cached_tokens are prompt
+ # tokens served from a prefix cache (cache reads); cache_write_tokens are
+ # tokens written into the cache on this call. Providers expose these under
+ # usage.prompt_tokens_details.cached_tokens / usage.cache_creation_input_tokens.
+ cached_tokens = db.Column(db.Integer, default=0)
+ cache_write_tokens = db.Column(db.Integer, default=0)
+
# Cost tracking (OpenRouter provides this)
cost = db.Column(db.Float, default=0.0)
diff --git a/src/services/llm.py b/src/services/llm.py
index 581e2fcd..4a3291dd 100644
--- a/src/services/llm.py
+++ b/src/services/llm.py
@@ -23,6 +23,49 @@ class TokenBudgetExceeded(Exception):
from src.utils import safe_json_loads, extract_json_object
+
+def extract_cache_tokens(usage):
+ """Pull prompt-cache counts out of a provider's usage object.
+
+ Returns (cached_tokens, cache_write_tokens) as ints, defaulting to 0.
+ Providers expose these differently:
+ - OpenAI / OpenRouter: usage.prompt_tokens_details.cached_tokens (reads),
+ and some surface a cache_write/cache_creation field alongside it.
+ - Anthropic: usage.cache_read_input_tokens / usage.cache_creation_input_tokens.
+ We probe all known shapes defensively because usage may be a pydantic
+ object or a plain dict, and missing fields must never break tracking.
+ """
+ if not usage:
+ return 0, 0
+
+ def _get(obj, name):
+ if obj is None:
+ return None
+ if isinstance(obj, dict):
+ return obj.get(name)
+ return getattr(obj, name, None)
+
+ def _to_int(val):
+ try:
+ return int(val) if val else 0
+ except (TypeError, ValueError):
+ return 0
+
+ details = _get(usage, 'prompt_tokens_details')
+
+ cached = (
+ _get(details, 'cached_tokens')
+ or _get(usage, 'cache_read_input_tokens')
+ or 0
+ )
+ cache_write = (
+ _get(usage, 'cache_creation_input_tokens')
+ or _get(details, 'cache_write_tokens')
+ or _get(usage, 'cache_write_tokens')
+ or 0
+ )
+ return _to_int(cached), _to_int(cache_write)
+
# Configuration - use TEXT_MODEL_* variables for LLM
TEXT_MODEL_API_KEY = os.environ.get("TEXT_MODEL_API_KEY")
TEXT_MODEL_BASE_URL = os.environ.get("TEXT_MODEL_BASE_URL", "https://openrouter.ai/api/v1")
@@ -261,6 +304,7 @@ def call_llm_completion(messages, temperature=0.7, response_format=None, stream=
if user_id and operation_type and not stream and response.usage:
try:
from src.services.token_tracking import token_tracker
+ cached_tokens, cache_write_tokens = extract_cache_tokens(response.usage)
token_tracker.record_usage(
user_id=user_id,
operation_type=operation_type,
@@ -268,7 +312,9 @@ def call_llm_completion(messages, temperature=0.7, response_format=None, stream=
completion_tokens=response.usage.completion_tokens,
total_tokens=response.usage.total_tokens,
model_name=TEXT_MODEL_NAME,
- cost=getattr(response.usage, 'cost', None)
+ cost=getattr(response.usage, 'cost', None),
+ cached_tokens=cached_tokens,
+ cache_write_tokens=cache_write_tokens
)
except Exception as e:
logger.warning(f"Failed to record token usage: {e}")
@@ -395,6 +441,7 @@ def call_chat_completion(messages, temperature=0.7, response_format=None, stream
if user_id and operation_type and not stream and response.usage:
try:
from src.services.token_tracking import token_tracker
+ cached_tokens, cache_write_tokens = extract_cache_tokens(response.usage)
token_tracker.record_usage(
user_id=user_id,
operation_type=operation_type,
@@ -402,7 +449,9 @@ def call_chat_completion(messages, temperature=0.7, response_format=None, stream
completion_tokens=response.usage.completion_tokens,
total_tokens=response.usage.total_tokens,
model_name=model_name,
- cost=getattr(response.usage, 'cost', None)
+ cost=getattr(response.usage, 'cost', None),
+ cached_tokens=cached_tokens,
+ cache_write_tokens=cache_write_tokens
)
except Exception as e:
logger.warning(f"Failed to record token usage: {e}")
@@ -479,6 +528,7 @@ def process_streaming_with_thinking(stream, user_id=None, operation_type=None, m
if hasattr(chunk, 'usage') and chunk.usage and user_id and operation_type:
try:
from src.services.token_tracking import token_tracker
+ cached_tokens, cache_write_tokens = extract_cache_tokens(chunk.usage)
# Use app context if provided (needed for generators where context may be lost)
if app:
with app.app_context():
@@ -489,7 +539,9 @@ def process_streaming_with_thinking(stream, user_id=None, operation_type=None, m
completion_tokens=chunk.usage.completion_tokens,
total_tokens=chunk.usage.total_tokens,
model_name=model_name or TEXT_MODEL_NAME,
- cost=getattr(chunk.usage, 'cost', None)
+ cost=getattr(chunk.usage, 'cost', None),
+ cached_tokens=cached_tokens,
+ cache_write_tokens=cache_write_tokens
)
else:
token_tracker.record_usage(
@@ -499,7 +551,9 @@ def process_streaming_with_thinking(stream, user_id=None, operation_type=None, m
completion_tokens=chunk.usage.completion_tokens,
total_tokens=chunk.usage.total_tokens,
model_name=model_name or TEXT_MODEL_NAME,
- cost=getattr(chunk.usage, 'cost', None)
+ cost=getattr(chunk.usage, 'cost', None),
+ cached_tokens=cached_tokens,
+ cache_write_tokens=cache_write_tokens
)
except Exception as e:
logger.warning(f"Failed to record streaming token usage: {e}")
diff --git a/src/services/token_tracking.py b/src/services/token_tracking.py
index 7bd39a67..03ddf48b 100644
--- a/src/services/token_tracking.py
+++ b/src/services/token_tracking.py
@@ -46,7 +46,9 @@ class TokenTracker:
completion_tokens: int,
total_tokens: int,
model_name: str = None,
- cost: float = None
+ cost: float = None,
+ cached_tokens: int = 0,
+ cache_write_tokens: int = 0
):
"""
Record token usage - upserts into daily aggregate.
@@ -59,6 +61,8 @@ class TokenTracker:
total_tokens: Total tokens (prompt + completion)
model_name: Name of the model used
cost: API cost if available (e.g., from OpenRouter)
+ cached_tokens: Prompt tokens served from a prefix cache (cache reads)
+ cache_write_tokens: Prompt tokens written into the cache on this call
"""
try:
today = date.today()
@@ -75,6 +79,8 @@ class TokenTracker:
usage.prompt_tokens += prompt_tokens
usage.completion_tokens += completion_tokens
usage.total_tokens += total_tokens
+ usage.cached_tokens = (usage.cached_tokens or 0) + (cached_tokens or 0)
+ usage.cache_write_tokens = (usage.cache_write_tokens or 0) + (cache_write_tokens or 0)
usage.request_count += 1
if cost:
usage.cost += cost
@@ -89,6 +95,8 @@ class TokenTracker:
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=total_tokens,
+ cached_tokens=cached_tokens or 0,
+ cache_write_tokens=cache_write_tokens or 0,
request_count=1,
model_name=model_name,
cost=cost or 0.0
@@ -180,7 +188,9 @@ class TokenTracker:
TokenUsage.date,
TokenUsage.operation_type,
func.sum(TokenUsage.total_tokens).label('tokens'),
- func.sum(TokenUsage.cost).label('cost')
+ func.sum(TokenUsage.cost).label('cost'),
+ func.sum(TokenUsage.cached_tokens).label('cached_tokens'),
+ func.sum(TokenUsage.cache_write_tokens).label('cache_write_tokens')
).filter(TokenUsage.date >= start_date)
if user_id:
@@ -201,12 +211,16 @@ class TokenTracker:
'llm_cost': 0.0,
'embedding_tokens': 0,
'embedding_cost': 0.0,
+ 'cached_tokens': 0,
+ 'cache_write_tokens': 0,
'by_operation': {},
}
tokens = r.tokens or 0
cost = r.cost or 0
stats[date_str]['total'] += tokens
stats[date_str]['cost'] += cost
+ stats[date_str]['cached_tokens'] += r.cached_tokens or 0
+ stats[date_str]['cache_write_tokens'] += r.cache_write_tokens or 0
if self.is_embedding_op(r.operation_type):
stats[date_str]['embedding_tokens'] += tokens
stats[date_str]['embedding_cost'] += cost
@@ -229,6 +243,8 @@ class TokenTracker:
'llm_cost': 0.0,
'embedding_tokens': 0,
'embedding_cost': 0.0,
+ 'cached_tokens': 0,
+ 'cache_write_tokens': 0,
'by_operation': {},
}
all_dates.append(date_str)
@@ -247,7 +263,9 @@ class TokenTracker:
extract('month', TokenUsage.date).label('month'),
TokenUsage.operation_type,
func.sum(TokenUsage.total_tokens).label('tokens'),
- func.sum(TokenUsage.cost).label('cost')
+ func.sum(TokenUsage.cost).label('cost'),
+ func.sum(TokenUsage.cached_tokens).label('cached_tokens'),
+ func.sum(TokenUsage.cache_write_tokens).label('cache_write_tokens')
).group_by('year', 'month', TokenUsage.operation_type).order_by('year', 'month').all()
buckets = {}
@@ -263,11 +281,15 @@ class TokenTracker:
'llm_cost': 0.0,
'embedding_tokens': 0,
'embedding_cost': 0.0,
+ 'cached_tokens': 0,
+ 'cache_write_tokens': 0,
}
tokens = r.tokens or 0
cost = r.cost or 0
buckets[key]['tokens'] += tokens
buckets[key]['cost'] += cost
+ buckets[key]['cached_tokens'] += r.cached_tokens or 0
+ buckets[key]['cache_write_tokens'] += r.cache_write_tokens or 0
if self.is_embedding_op(r.operation_type):
buckets[key]['embedding_tokens'] += tokens
buckets[key]['embedding_cost'] += cost
@@ -315,7 +337,9 @@ class TokenTracker:
query = db.session.query(
TokenUsage.operation_type,
func.sum(TokenUsage.total_tokens).label('tokens'),
- func.sum(TokenUsage.cost).label('cost')
+ func.sum(TokenUsage.cost).label('cost'),
+ func.sum(TokenUsage.cached_tokens).label('cached_tokens'),
+ func.sum(TokenUsage.cache_write_tokens).label('cache_write_tokens')
).filter(TokenUsage.date == today)
if user_id:
@@ -330,12 +354,16 @@ class TokenTracker:
'llm_cost': 0.0,
'embedding_tokens': 0,
'embedding_cost': 0.0,
+ 'cached_tokens': 0,
+ 'cache_write_tokens': 0,
}
for r in rows:
tokens = r.tokens or 0
cost = r.cost or 0
out['tokens'] += tokens
out['cost'] += cost
+ out['cached_tokens'] += r.cached_tokens or 0
+ out['cache_write_tokens'] += r.cache_write_tokens or 0
if self.is_embedding_op(r.operation_type):
out['embedding_tokens'] += tokens
out['embedding_cost'] += cost
diff --git a/templates/admin.html b/templates/admin.html
index 4353eecf..8dbb6dde 100644
--- a/templates/admin.html
+++ b/templates/admin.html
@@ -462,6 +462,11 @@
<p class="text-lg font-semibold text-[var(--text-primary)]">$${ (tokenStats.current_month?.llm_cost || 0).toFixed(4) }</p>
</div>
</div>
+ <!-- Prompt cache: reads are prompt tokens served from a prefix cache. -->
+ <p class="text-xs text-[var(--text-muted)] mt-2 flex items-center flex-wrap gap-x-3">
+ <span><i class="fas fa-bolt mr-1 text-[var(--text-accent)]"></i>Cache reads: <span class="font-semibold text-[var(--text-secondary)]">${ (tokenStats.current_month?.cached_tokens || 0).toLocaleString() }</span> this month <span v-if="(tokenStats.today?.cached_tokens || 0) > 0">(${ (tokenStats.today?.cached_tokens || 0).toLocaleString() } today)</span></span>
+ <span v-if="(tokenStats.current_month?.cache_write_tokens || 0) > 0">· writes: <span class="font-semibold text-[var(--text-secondary)]">${ (tokenStats.current_month?.cache_write_tokens || 0).toLocaleString() }</span></span>
+ </p>
</div>
<!-- Embedding bucket -->
@@ -516,6 +521,9 @@
<p class="text-xs text-[var(--text-muted)]">
${ op.requests } req<span v-if="op.cost > 0"> · $${ op.cost.toFixed(4) }</span>
</p>
+ <p v-if="(op.cached_tokens || 0) > 0" class="text-xs text-[var(--text-accent)]">
+ <i class="fas fa-bolt mr-1"></i>${ op.cached_tokens.toLocaleString() } cached
+ </p>
</div>
</div>
</div>
diff --git a/tests/test_cov_llm.py b/tests/test_cov_llm.py
index 6a185189..8b3cbd68 100644
--- a/tests/test_cov_llm.py
+++ b/tests/test_cov_llm.py
@@ -30,6 +30,7 @@ from src.services.llm import (
call_chat_completion,
format_api_error_message,
process_streaming_with_thinking,
+ extract_cache_tokens,
TokenBudgetExceeded,
)
@@ -332,6 +333,74 @@ def test_call_llm_completion_records_usage(monkeypatch):
assert rec_kwargs["completion_tokens"] == 7
assert rec_kwargs["total_tokens"] == 18
assert rec_kwargs["cost"] == 0.001
+ # No cache fields on this usage object → defaults to 0.
+ assert rec_kwargs["cached_tokens"] == 0
+ assert rec_kwargs["cache_write_tokens"] == 0
+
+
+def test_call_llm_completion_records_cache_tokens(monkeypatch):
+ """When the provider reports prefix-cache reads/writes, they reach the
+ tracker so the admin dashboard can show them."""
+ usage = make_usage(prompt=8000, completion=20, total=8020)
+ usage.prompt_tokens_details = SimpleNamespace(cached_tokens=6000)
+ usage.cache_creation_input_tokens = 1500
+ install_client(monkeypatch, return_value=make_completion("ok", usage=usage))
+ monkeypatch.setattr(llm, "TEXT_MODEL_API_KEY", "test-key")
+ monkeypatch.setattr(llm, "TEXT_MODEL_NAME", "gpt-4o")
+ fake_tracker = MagicMock()
+ fake_tracker.check_budget.return_value = (True, 50.0, None)
+ with patch("src.services.token_tracking.token_tracker", fake_tracker):
+ call_llm_completion(
+ [{"role": "user", "content": "q"}],
+ user_id=99012,
+ operation_type="summarization",
+ )
+ rec_kwargs = fake_tracker.record_usage.call_args.kwargs
+ assert rec_kwargs["cached_tokens"] == 6000
+ assert rec_kwargs["cache_write_tokens"] == 1500
+
+
+# ---------------------------------------------------------------------------
+# extract_cache_tokens — provider shape probing
+# ---------------------------------------------------------------------------
+
+def test_extract_cache_tokens_none_usage():
+ assert extract_cache_tokens(None) == (0, 0)
+
+
+def test_extract_cache_tokens_openai_shape():
+ usage = SimpleNamespace(
+ prompt_tokens_details=SimpleNamespace(cached_tokens=4096)
+ )
+ assert extract_cache_tokens(usage) == (4096, 0)
+
+
+def test_extract_cache_tokens_anthropic_shape():
+ usage = SimpleNamespace(
+ cache_read_input_tokens=2048,
+ cache_creation_input_tokens=512,
+ )
+ assert extract_cache_tokens(usage) == (2048, 512)
+
+
+def test_extract_cache_tokens_dict_shape():
+ usage = {
+ "prompt_tokens_details": {"cached_tokens": 100, "cache_write_tokens": 30},
+ }
+ assert extract_cache_tokens(usage) == (100, 30)
+
+
+def test_extract_cache_tokens_missing_fields_default_zero():
+ # A usage object with the standard fields but no cache details.
+ assert extract_cache_tokens(make_usage()) == (0, 0)
+
+
+def test_extract_cache_tokens_handles_garbage_values():
+ usage = SimpleNamespace(
+ prompt_tokens_details=SimpleNamespace(cached_tokens="not-a-number"),
+ cache_creation_input_tokens=None,
+ )
+ assert extract_cache_tokens(usage) == (0, 0)
def test_call_llm_completion_budget_warning_near_limit(monkeypatch):
diff --git a/tests/test_cov_token_tracking_cache.py b/tests/test_cov_token_tracking_cache.py
new file mode 100644
index 00000000..3aa730e1
--- /dev/null
+++ b/tests/test_cov_token_tracking_cache.py
@@ -0,0 +1,87 @@
+"""
+Coverage for prompt-cache token tracking in src/services/token_tracking.py.
+
+record_usage() now accepts cached_tokens / cache_write_tokens (subsets of the
+prompt tokens that a prefix-caching backend served from / wrote to cache).
+These must accumulate into the daily aggregate row and surface through
+get_today_usage().
+
+The test DB is shared across the suite, so every assertion is scoped to a
+freshly created unique user — never to global aggregates.
+"""
+
+import pytest
+
+from src.app import app, db
+from src.models import User
+from src.models.token_usage import TokenUsage
+from src.services.token_tracking import token_tracker
+
+
+def isolated_user():
+ with app.app_context():
+ user = User(username="cache_track_test_user", email="[email protected]")
+ user.password = "unused"
+ db.session.add(user)
+ db.session.commit()
+ uid = user.id
+ try:
+ yield uid
+ finally:
+ TokenUsage.query.filter_by(user_id=uid).delete()
+ db.session.query(User).filter_by(id=uid).delete()
+ db.session.commit()
+
+
+def test_record_usage_accumulates_cache_tokens(isolated_user):
+ with app.app_context():
+ token_tracker.record_usage(
+ user_id=isolated_user,
+ operation_type="summarization",
+ prompt_tokens=8000, completion_tokens=20, total_tokens=8020,
+ cached_tokens=6000, cache_write_tokens=1500,
+ )
+ # Second call on the same day/operation must add to the existing row.
+ token_tracker.record_usage(
+ user_id=isolated_user,
+ operation_type="summarization",
+ prompt_tokens=8000, completion_tokens=20, total_tokens=8020,
+ cached_tokens=7000, cache_write_tokens=0,
+ )
+
+ row = TokenUsage.query.filter_by(
+ user_id=isolated_user, operation_type="summarization"
+ ).first()
+ assert row is not None
+ assert row.cached_tokens == 13000
+ assert row.cache_write_tokens == 1500
+ assert row.request_count == 2
+
+
+def test_get_today_usage_surfaces_cache_tokens(isolated_user):
+ with app.app_context():
+ token_tracker.record_usage(
+ user_id=isolated_user,
+ operation_type="chat",
+ prompt_tokens=500, completion_tokens=10, total_tokens=510,
+ cached_tokens=300, cache_write_tokens=120,
+ )
+ today = token_tracker.get_today_usage(user_id=isolated_user)
+ assert today["cached_tokens"] == 300
+ assert today["cache_write_tokens"] == 120
+
+
+def test_record_usage_defaults_cache_tokens_to_zero(isolated_user):
+ """Existing callers that pass no cache args must still work and store 0."""
+ with app.app_context():
+ token_tracker.record_usage(
+ user_id=isolated_user,
+ operation_type="title_generation",
+ prompt_tokens=100, completion_tokens=5, total_tokens=105,
+ )
+ row = TokenUsage.query.filter_by(
+ user_id=isolated_user, operation_type="title_generation"
+ ).first()
+ assert row.cached_tokens == 0
+ assert row.cache_write_tokens == 0