Speakr is a personal, self-hosted web application designed for transcribing audio recordings
Python
Track and display prompt-cache token usage (cache reads/writes) in admin token stats
Murtaza Nasirc884f16238ab
diff --git a/src/api/admin.py b/src/api/admin.pyindex bf75c796..00a97ac3 100644--- a/src/api/admin.py+++ b/src/api/admin.py@@ -497,6 +497,8 @@ def admin_get_token_stats():func.sum(TokenUsage.total_tokens).label('tokens'),func.sum(TokenUsage.cost).label('cost'),func.sum(TokenUsage.request_count).label('requests'),+ func.sum(TokenUsage.cached_tokens).label('cached_tokens'),+ func.sum(TokenUsage.cache_write_tokens).label('cache_write_tokens'),).filter(extract('year', TokenUsage.date) == today.year,extract('month', TokenUsage.date) == today.month,@@ -507,6 +509,8 @@ def admin_get_token_stats():'tokens': int(r.tokens or 0),'cost': float(r.cost or 0.0),'requests': int(r.requests or 0),+ 'cached_tokens': int(r.cached_tokens or 0),+ 'cache_write_tokens': int(r.cache_write_tokens or 0),'is_embedding': token_tracker.is_embedding_op(r.operation_type),}for r in op_rows@@ -521,6 +525,8 @@ def admin_get_token_stats():'llm_cost': current_month.get('llm_cost', 0),'embedding_tokens': current_month.get('embedding_tokens', 0),'embedding_cost': current_month.get('embedding_cost', 0),+ 'cached_tokens': current_month.get('cached_tokens', 0),+ 'cache_write_tokens': current_month.get('cache_write_tokens', 0),'by_operation': by_operation,},'user_count_with_usage': len([u for u in user_stats if u['current_usage'] > 0]),diff --git a/src/init_db.py b/src/init_db.pyindex ce4e5260..0333d4ff 100644--- a/src/init_db.py+++ b/src/init_db.py@@ -233,6 +233,13 @@ def initialize_database(app):if add_column_if_not_exists(engine, 'initial_prompt_template', 'hotwords', 'TEXT'):app.logger.info("Added hotwords column to initial_prompt_template table")+ # Prompt-cache token tracking: cached_tokens (prefix-cache reads) and+ # cache_write_tokens (tokens written to the cache) per usage row.+ if add_column_if_not_exists(engine, 'token_usage', 'cached_tokens', 'INTEGER DEFAULT 0'):+ app.logger.info("Added cached_tokens column to token_usage table")+ if add_column_if_not_exists(engine, 'token_usage', 'cache_write_tokens', 'INTEGER DEFAULT 0'):+ app.logger.info("Added cache_write_tokens column to token_usage table")+# Add speaker voice profile embedding fieldsif add_column_if_not_exists(engine, 'speaker', 'average_embedding', 'BLOB'):app.logger.info("Added average_embedding column to speaker table")diff --git a/src/models/token_usage.py b/src/models/token_usage.pyindex cd7461fd..59e2be1e 100644--- a/src/models/token_usage.py+++ b/src/models/token_usage.py@@ -20,6 +20,13 @@ class TokenUsage(db.Model):completion_tokens = db.Column(db.Integer, default=0)total_tokens = db.Column(db.Integer, default=0)+ # Prompt-cache counts (subset of prompt_tokens). cached_tokens are prompt+ # tokens served from a prefix cache (cache reads); cache_write_tokens are+ # tokens written into the cache on this call. Providers expose these under+ # usage.prompt_tokens_details.cached_tokens / usage.cache_creation_input_tokens.+ cached_tokens = db.Column(db.Integer, default=0)+ cache_write_tokens = db.Column(db.Integer, default=0)+# Cost tracking (OpenRouter provides this)cost = db.Column(db.Float, default=0.0)diff --git a/src/services/llm.py b/src/services/llm.pyindex 581e2fcd..4a3291dd 100644--- a/src/services/llm.py+++ b/src/services/llm.py@@ -23,6 +23,49 @@ class TokenBudgetExceeded(Exception):from src.utils import safe_json_loads, extract_json_object++def extract_cache_tokens(usage):+ """Pull prompt-cache counts out of a provider's usage object.++ Returns (cached_tokens, cache_write_tokens) as ints, defaulting to 0.+ Providers expose these differently:+ - OpenAI / OpenRouter: usage.prompt_tokens_details.cached_tokens (reads),+ and some surface a cache_write/cache_creation field alongside it.+ - Anthropic: usage.cache_read_input_tokens / usage.cache_creation_input_tokens.+ We probe all known shapes defensively because usage may be a pydantic+ object or a plain dict, and missing fields must never break tracking.+ """+ if not usage:+ return 0, 0++ def _get(obj, name):+ if obj is None:+ return None+ if isinstance(obj, dict):+ return obj.get(name)+ return getattr(obj, name, None)++ def _to_int(val):+ try:+ return int(val) if val else 0+ except (TypeError, ValueError):+ return 0++ details = _get(usage, 'prompt_tokens_details')++ cached = (+ _get(details, 'cached_tokens')+ or _get(usage, 'cache_read_input_tokens')+ or 0+ )+ cache_write = (+ _get(usage, 'cache_creation_input_tokens')+ or _get(details, 'cache_write_tokens')+ or _get(usage, 'cache_write_tokens')+ or 0+ )+ return _to_int(cached), _to_int(cache_write)+# Configuration - use TEXT_MODEL_* variables for LLMTEXT_MODEL_API_KEY = os.environ.get("TEXT_MODEL_API_KEY")TEXT_MODEL_BASE_URL = os.environ.get("TEXT_MODEL_BASE_URL", "https://openrouter.ai/api/v1")@@ -261,6 +304,7 @@ def call_llm_completion(messages, temperature=0.7, response_format=None, stream=if user_id and operation_type and not stream and response.usage:try:from src.services.token_tracking import token_tracker+ cached_tokens, cache_write_tokens = extract_cache_tokens(response.usage)token_tracker.record_usage(user_id=user_id,operation_type=operation_type,@@ -268,7 +312,9 @@ def call_llm_completion(messages, temperature=0.7, response_format=None, stream=completion_tokens=response.usage.completion_tokens,total_tokens=response.usage.total_tokens,model_name=TEXT_MODEL_NAME,- cost=getattr(response.usage, 'cost', None)+ cost=getattr(response.usage, 'cost', None),+ cached_tokens=cached_tokens,+ cache_write_tokens=cache_write_tokens)except Exception as e:logger.warning(f"Failed to record token usage: {e}")@@ -395,6 +441,7 @@ def call_chat_completion(messages, temperature=0.7, response_format=None, streamif user_id and operation_type and not stream and response.usage:try:from src.services.token_tracking import token_tracker+ cached_tokens, cache_write_tokens = extract_cache_tokens(response.usage)token_tracker.record_usage(user_id=user_id,operation_type=operation_type,@@ -402,7 +449,9 @@ def call_chat_completion(messages, temperature=0.7, response_format=None, streamcompletion_tokens=response.usage.completion_tokens,total_tokens=response.usage.total_tokens,model_name=model_name,- cost=getattr(response.usage, 'cost', None)+ cost=getattr(response.usage, 'cost', None),+ cached_tokens=cached_tokens,+ cache_write_tokens=cache_write_tokens)except Exception as e:logger.warning(f"Failed to record token usage: {e}")@@ -479,6 +528,7 @@ def process_streaming_with_thinking(stream, user_id=None, operation_type=None, mif hasattr(chunk, 'usage') and chunk.usage and user_id and operation_type:try:from src.services.token_tracking import token_tracker+ cached_tokens, cache_write_tokens = extract_cache_tokens(chunk.usage)# Use app context if provided (needed for generators where context may be lost)if app:with app.app_context():@@ -489,7 +539,9 @@ def process_streaming_with_thinking(stream, user_id=None, operation_type=None, mcompletion_tokens=chunk.usage.completion_tokens,total_tokens=chunk.usage.total_tokens,model_name=model_name or TEXT_MODEL_NAME,- cost=getattr(chunk.usage, 'cost', None)+ cost=getattr(chunk.usage, 'cost', None),+ cached_tokens=cached_tokens,+ cache_write_tokens=cache_write_tokens)else:token_tracker.record_usage(@@ -499,7 +551,9 @@ def process_streaming_with_thinking(stream, user_id=None, operation_type=None, mcompletion_tokens=chunk.usage.completion_tokens,total_tokens=chunk.usage.total_tokens,model_name=model_name or TEXT_MODEL_NAME,- cost=getattr(chunk.usage, 'cost', None)+ cost=getattr(chunk.usage, 'cost', None),+ cached_tokens=cached_tokens,+ cache_write_tokens=cache_write_tokens)except Exception as e:logger.warning(f"Failed to record streaming token usage: {e}")diff --git a/src/services/token_tracking.py b/src/services/token_tracking.pyindex 7bd39a67..03ddf48b 100644--- a/src/services/token_tracking.py+++ b/src/services/token_tracking.py@@ -46,7 +46,9 @@ class TokenTracker:completion_tokens: int,total_tokens: int,model_name: str = None,- cost: float = None+ cost: float = None,+ cached_tokens: int = 0,+ cache_write_tokens: int = 0):"""Record token usage - upserts into daily aggregate.@@ -59,6 +61,8 @@ class TokenTracker:total_tokens: Total tokens (prompt + completion)model_name: Name of the model usedcost: API cost if available (e.g., from OpenRouter)+ cached_tokens: Prompt tokens served from a prefix cache (cache reads)+ cache_write_tokens: Prompt tokens written into the cache on this call"""try:today = date.today()@@ -75,6 +79,8 @@ class TokenTracker:usage.prompt_tokens += prompt_tokensusage.completion_tokens += completion_tokensusage.total_tokens += total_tokens+ usage.cached_tokens = (usage.cached_tokens or 0) + (cached_tokens or 0)+ usage.cache_write_tokens = (usage.cache_write_tokens or 0) + (cache_write_tokens or 0)usage.request_count += 1if cost:usage.cost += cost@@ -89,6 +95,8 @@ class TokenTracker:prompt_tokens=prompt_tokens,completion_tokens=completion_tokens,total_tokens=total_tokens,+ cached_tokens=cached_tokens or 0,+ cache_write_tokens=cache_write_tokens or 0,request_count=1,model_name=model_name,cost=cost or 0.0@@ -180,7 +188,9 @@ class TokenTracker:TokenUsage.date,TokenUsage.operation_type,func.sum(TokenUsage.total_tokens).label('tokens'),- func.sum(TokenUsage.cost).label('cost')+ func.sum(TokenUsage.cost).label('cost'),+ func.sum(TokenUsage.cached_tokens).label('cached_tokens'),+ func.sum(TokenUsage.cache_write_tokens).label('cache_write_tokens')).filter(TokenUsage.date >= start_date)if user_id:@@ -201,12 +211,16 @@ class TokenTracker:'llm_cost': 0.0,'embedding_tokens': 0,'embedding_cost': 0.0,+ 'cached_tokens': 0,+ 'cache_write_tokens': 0,'by_operation': {},}tokens = r.tokens or 0cost = r.cost or 0stats[date_str]['total'] += tokensstats[date_str]['cost'] += cost+ stats[date_str]['cached_tokens'] += r.cached_tokens or 0+ stats[date_str]['cache_write_tokens'] += r.cache_write_tokens or 0if self.is_embedding_op(r.operation_type):stats[date_str]['embedding_tokens'] += tokensstats[date_str]['embedding_cost'] += cost@@ -229,6 +243,8 @@ class TokenTracker:'llm_cost': 0.0,'embedding_tokens': 0,'embedding_cost': 0.0,+ 'cached_tokens': 0,+ 'cache_write_tokens': 0,'by_operation': {},}all_dates.append(date_str)@@ -247,7 +263,9 @@ class TokenTracker:extract('month', TokenUsage.date).label('month'),TokenUsage.operation_type,func.sum(TokenUsage.total_tokens).label('tokens'),- func.sum(TokenUsage.cost).label('cost')+ func.sum(TokenUsage.cost).label('cost'),+ func.sum(TokenUsage.cached_tokens).label('cached_tokens'),+ func.sum(TokenUsage.cache_write_tokens).label('cache_write_tokens')).group_by('year', 'month', TokenUsage.operation_type).order_by('year', 'month').all()buckets = {}@@ -263,11 +281,15 @@ class TokenTracker:'llm_cost': 0.0,'embedding_tokens': 0,'embedding_cost': 0.0,+ 'cached_tokens': 0,+ 'cache_write_tokens': 0,}tokens = r.tokens or 0cost = r.cost or 0buckets[key]['tokens'] += tokensbuckets[key]['cost'] += cost+ buckets[key]['cached_tokens'] += r.cached_tokens or 0+ buckets[key]['cache_write_tokens'] += r.cache_write_tokens or 0if self.is_embedding_op(r.operation_type):buckets[key]['embedding_tokens'] += tokensbuckets[key]['embedding_cost'] += cost@@ -315,7 +337,9 @@ class TokenTracker:query = db.session.query(TokenUsage.operation_type,func.sum(TokenUsage.total_tokens).label('tokens'),- func.sum(TokenUsage.cost).label('cost')+ func.sum(TokenUsage.cost).label('cost'),+ func.sum(TokenUsage.cached_tokens).label('cached_tokens'),+ func.sum(TokenUsage.cache_write_tokens).label('cache_write_tokens')).filter(TokenUsage.date == today)if user_id:@@ -330,12 +354,16 @@ class TokenTracker:'llm_cost': 0.0,'embedding_tokens': 0,'embedding_cost': 0.0,+ 'cached_tokens': 0,+ 'cache_write_tokens': 0,}for r in rows:tokens = r.tokens or 0cost = r.cost or 0out['tokens'] += tokensout['cost'] += cost+ out['cached_tokens'] += r.cached_tokens or 0+ out['cache_write_tokens'] += r.cache_write_tokens or 0if self.is_embedding_op(r.operation_type):out['embedding_tokens'] += tokensout['embedding_cost'] += costdiff --git a/templates/admin.html b/templates/admin.htmlindex 4353eecf..8dbb6dde 100644--- a/templates/admin.html+++ b/templates/admin.html@@ -462,6 +462,11 @@<p class="text-lg font-semibold text-[var(--text-primary)]">$${ (tokenStats.current_month?.llm_cost || 0).toFixed(4) }</p></div></div>+ <!-- Prompt cache: reads are prompt tokens served from a prefix cache. -->+ <p class="text-xs text-[var(--text-muted)] mt-2 flex items-center flex-wrap gap-x-3">+ <span><i class="fas fa-bolt mr-1 text-[var(--text-accent)]"></i>Cache reads: <span class="font-semibold text-[var(--text-secondary)]">${ (tokenStats.current_month?.cached_tokens || 0).toLocaleString() }</span> this month <span v-if="(tokenStats.today?.cached_tokens || 0) > 0">(${ (tokenStats.today?.cached_tokens || 0).toLocaleString() } today)</span></span>+ <span v-if="(tokenStats.current_month?.cache_write_tokens || 0) > 0">· writes: <span class="font-semibold text-[var(--text-secondary)]">${ (tokenStats.current_month?.cache_write_tokens || 0).toLocaleString() }</span></span>+ </p></div><!-- Embedding bucket -->@@ -516,6 +521,9 @@<p class="text-xs text-[var(--text-muted)]">${ op.requests } req<span v-if="op.cost > 0"> · $${ op.cost.toFixed(4) }</span></p>+ <p v-if="(op.cached_tokens || 0) > 0" class="text-xs text-[var(--text-accent)]">+ <i class="fas fa-bolt mr-1"></i>${ op.cached_tokens.toLocaleString() } cached+ </p></div></div></div>diff --git a/tests/test_cov_llm.py b/tests/test_cov_llm.pyindex 6a185189..8b3cbd68 100644--- a/tests/test_cov_llm.py+++ b/tests/test_cov_llm.py@@ -30,6 +30,7 @@ from src.services.llm import (call_chat_completion,format_api_error_message,process_streaming_with_thinking,+ extract_cache_tokens,TokenBudgetExceeded,)@@ -332,6 +333,74 @@ def test_call_llm_completion_records_usage(monkeypatch):assert rec_kwargs["completion_tokens"] == 7assert rec_kwargs["total_tokens"] == 18assert rec_kwargs["cost"] == 0.001+ # No cache fields on this usage object → defaults to 0.+ assert rec_kwargs["cached_tokens"] == 0+ assert rec_kwargs["cache_write_tokens"] == 0+++def test_call_llm_completion_records_cache_tokens(monkeypatch):+ """When the provider reports prefix-cache reads/writes, they reach the+ tracker so the admin dashboard can show them."""+ usage = make_usage(prompt=8000, completion=20, total=8020)+ usage.prompt_tokens_details = SimpleNamespace(cached_tokens=6000)+ usage.cache_creation_input_tokens = 1500+ install_client(monkeypatch, return_value=make_completion("ok", usage=usage))+ monkeypatch.setattr(llm, "TEXT_MODEL_API_KEY", "test-key")+ monkeypatch.setattr(llm, "TEXT_MODEL_NAME", "gpt-4o")+ fake_tracker = MagicMock()+ fake_tracker.check_budget.return_value = (True, 50.0, None)+ with patch("src.services.token_tracking.token_tracker", fake_tracker):+ call_llm_completion(+ [{"role": "user", "content": "q"}],+ user_id=99012,+ operation_type="summarization",+ )+ rec_kwargs = fake_tracker.record_usage.call_args.kwargs+ assert rec_kwargs["cached_tokens"] == 6000+ assert rec_kwargs["cache_write_tokens"] == 1500+++# ---------------------------------------------------------------------------+# extract_cache_tokens — provider shape probing+# ---------------------------------------------------------------------------++def test_extract_cache_tokens_none_usage():+ assert extract_cache_tokens(None) == (0, 0)+++def test_extract_cache_tokens_openai_shape():+ usage = SimpleNamespace(+ prompt_tokens_details=SimpleNamespace(cached_tokens=4096)+ )+ assert extract_cache_tokens(usage) == (4096, 0)+++def test_extract_cache_tokens_anthropic_shape():+ usage = SimpleNamespace(+ cache_read_input_tokens=2048,+ cache_creation_input_tokens=512,+ )+ assert extract_cache_tokens(usage) == (2048, 512)+++def test_extract_cache_tokens_dict_shape():+ usage = {+ "prompt_tokens_details": {"cached_tokens": 100, "cache_write_tokens": 30},+ }+ assert extract_cache_tokens(usage) == (100, 30)+++def test_extract_cache_tokens_missing_fields_default_zero():+ # A usage object with the standard fields but no cache details.+ assert extract_cache_tokens(make_usage()) == (0, 0)+++def test_extract_cache_tokens_handles_garbage_values():+ usage = SimpleNamespace(+ prompt_tokens_details=SimpleNamespace(cached_tokens="not-a-number"),+ cache_creation_input_tokens=None,+ )+ assert extract_cache_tokens(usage) == (0, 0)def test_call_llm_completion_budget_warning_near_limit(monkeypatch):diff --git a/tests/test_cov_token_tracking_cache.py b/tests/test_cov_token_tracking_cache.pynew file mode 100644index 00000000..3aa730e1--- /dev/null+++ b/tests/test_cov_token_tracking_cache.py@@ -0,0 +1,87 @@+"""+Coverage for prompt-cache token tracking in src/services/token_tracking.py.++record_usage() now accepts cached_tokens / cache_write_tokens (subsets of the+prompt tokens that a prefix-caching backend served from / wrote to cache).+These must accumulate into the daily aggregate row and surface through+get_today_usage().++The test DB is shared across the suite, so every assertion is scoped to a+freshly created unique user — never to global aggregates.+"""++import pytest++from src.app import app, db+from src.models import User+from src.models.token_usage import TokenUsage+from src.services.token_tracking import token_tracker+++def isolated_user():+ with app.app_context():+ user = User(username="cache_track_test_user", email="[email protected]")+ user.password = "unused"+ db.session.add(user)+ db.session.commit()+ uid = user.id+ try:+ yield uid+ finally:+ TokenUsage.query.filter_by(user_id=uid).delete()+ db.session.query(User).filter_by(id=uid).delete()+ db.session.commit()+++def test_record_usage_accumulates_cache_tokens(isolated_user):+ with app.app_context():+ token_tracker.record_usage(+ user_id=isolated_user,+ operation_type="summarization",+ prompt_tokens=8000, completion_tokens=20, total_tokens=8020,+ cached_tokens=6000, cache_write_tokens=1500,+ )+ # Second call on the same day/operation must add to the existing row.+ token_tracker.record_usage(+ user_id=isolated_user,+ operation_type="summarization",+ prompt_tokens=8000, completion_tokens=20, total_tokens=8020,+ cached_tokens=7000, cache_write_tokens=0,+ )++ row = TokenUsage.query.filter_by(+ user_id=isolated_user, operation_type="summarization"+ ).first()+ assert row is not None+ assert row.cached_tokens == 13000+ assert row.cache_write_tokens == 1500+ assert row.request_count == 2+++def test_get_today_usage_surfaces_cache_tokens(isolated_user):+ with app.app_context():+ token_tracker.record_usage(+ user_id=isolated_user,+ operation_type="chat",+ prompt_tokens=500, completion_tokens=10, total_tokens=510,+ cached_tokens=300, cache_write_tokens=120,+ )+ today = token_tracker.get_today_usage(user_id=isolated_user)+ assert today["cached_tokens"] == 300+ assert today["cache_write_tokens"] == 120+++def test_record_usage_defaults_cache_tokens_to_zero(isolated_user):+ """Existing callers that pass no cache args must still work and store 0."""+ with app.app_context():+ token_tracker.record_usage(+ user_id=isolated_user,+ operation_type="title_generation",+ prompt_tokens=100, completion_tokens=5, total_tokens=105,+ )+ row = TokenUsage.query.filter_by(+ user_id=isolated_user, operation_type="title_generation"+ ).first()+ assert row.cached_tokens == 0+ assert row.cache_write_tokens == 0