Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(metrics): record LLM calls when provider reports no token usage (#4473)

LearningCircuit66754e83782f
diff --git a/changelog.d/4473.bugfix.md b/changelog.d/4473.bugfix.md
new file mode 100644
index 0000000000..2442507895
--- /dev/null
+++ b/changelog.d/4473.bugfix.md
@@ -0,0 +1 @@
+Fixed metrics pages showing "No token usage data yet" despite completed researches (#4457). LLM calls whose provider returns no token usage data (OpenAI-compatible servers that omit `usage` on streamed responses, proxies that strip it, Ollama omitting `prompt_eval_count` for fully-cached prompts) were silently skipped, leaving every metrics panel empty; they are now recorded with zero token counts plus a warning log, so call counts, models, phases and response times still appear. Added an opt-in `llm.openai_endpoint.stream_usage` setting that requests usage stats on streamed responses (`stream_options.include_usage`) for servers that support it (LM Studio 0.3.18+, llama.cpp, vLLM, OpenRouter) — off by default because some gateways reject the parameter with a 400. Also fixed background-thread token writes not updating the `ModelUsage` aggregate, and several /metrics panels ignoring the selected time period: the research count, plus the ratings, link and strategy analytics, fell back to a fixed 30-day (or all-time) window when "3 months" or "1 year" was chosen, because the period vocabularies were inconsistent. All metrics endpoints now share one period map.
diff --git a/src/local_deep_research/database/thread_metrics.py b/src/local_deep_research/database/thread_metrics.py
index ca9738274a..261232aee5 100644
--- a/src/local_deep_research/database/thread_metrics.py
+++ b/src/local_deep_research/database/thread_metrics.py
@@ -128,7 +128,7 @@ class ThreadSafeMetricsWriter:
"""
with self.get_session(username) as session:
# Import here to avoid circular imports
- from .models import TokenUsage
+ from .models import ModelUsage, TokenUsage
# Create TokenUsage record
token_usage = TokenUsage(
@@ -174,6 +174,30 @@ class ThreadSafeMetricsWriter:
)
session.add(token_usage)
+ # Update or create model usage statistics (matches MainThread
+ # path in token_counter.py _save_to_db).
+ total_tokens = token_data.get("prompt_tokens", 0) + token_data.get(
+ "completion_tokens", 0
+ )
+ model_name = token_data.get("model_name")
+ model_usage = (
+ session.query(ModelUsage)
+ .filter_by(model_name=model_name)
+ .first()
+ )
+
+ if model_usage:
+ model_usage.total_tokens += total_tokens
+ model_usage.total_calls += 1
+ else:
+ model_usage = ModelUsage(
+ model_name=model_name,
+ model_provider=token_data.get("provider"),
+ total_tokens=total_tokens,
+ total_calls=1,
+ )
+ session.add(model_usage)
+
# Global instance for thread-safe metrics
metrics_writer = ThreadSafeMetricsWriter()
diff --git a/src/local_deep_research/defaults/default_settings.json b/src/local_deep_research/defaults/default_settings.json
index 8ae357bcf6..3cc3c83d25 100644
--- a/src/local_deep_research/defaults/default_settings.json
+++ b/src/local_deep_research/defaults/default_settings.json
@@ -596,6 +596,20 @@
"value": "https://openrouter.ai/api/v1",
"visible": true
},
+ "llm.openai_endpoint.stream_usage": {
+ "category": null,
+ "description": "Request token usage statistics on streamed responses (sends stream_options.include_usage). Enable this if the token metrics dashboard shows zero token counts for streamed calls. Supported by LM Studio (0.3.18+), llama.cpp server, vLLM, and OpenRouter. Leave disabled if your endpoint rejects requests with a 'stream_options' error (e.g. some API gateways).",
+ "editable": true,
+ "max_value": null,
+ "min_value": null,
+ "name": "Stream Token Usage",
+ "options": null,
+ "step": null,
+ "type": "LLM",
+ "ui_element": "checkbox",
+ "value": false,
+ "visible": true
+ },
"llm.ollama.api_key": {
"category": null,
"description": "Optional API key for Ollama if you've put it behind a proxy that requires authentication. Leave empty for the default no-auth setup.",
diff --git a/src/local_deep_research/llm/providers/implementations/custom_openai_endpoint.py b/src/local_deep_research/llm/providers/implementations/custom_openai_endpoint.py
index e9b5b9ba92..e30d60377c 100644
--- a/src/local_deep_research/llm/providers/implementations/custom_openai_endpoint.py
+++ b/src/local_deep_research/llm/providers/implementations/custom_openai_endpoint.py
@@ -110,4 +110,16 @@ class CustomOpenAIEndpointProvider(OpenAICompatibleProvider):
normalize_url(custom_url) if custom_url else cls.default_base_url
)
+ # Opt-in token usage on streamed responses. Off by default because
+ # some OpenAI-compatible gateways reject stream_options with a 400
+ # (e.g. xAI, Databricks AI Gateway, Azure "on your data"), which
+ # would break the call entirely — worse than missing token counts.
+ stream_usage = get_setting_from_snapshot(
+ "llm.openai_endpoint.stream_usage",
+ default=False,
+ settings_snapshot=settings_snapshot,
+ )
+ if stream_usage:
+ kwargs["stream_usage"] = True
+
return super().create_llm(model_name, temperature, **kwargs)
diff --git a/src/local_deep_research/llm/providers/openai_base.py b/src/local_deep_research/llm/providers/openai_base.py
index 848d90bf72..3dc6921d3b 100644
--- a/src/local_deep_research/llm/providers/openai_base.py
+++ b/src/local_deep_research/llm/providers/openai_base.py
@@ -134,6 +134,12 @@ class OpenAICompatibleProvider(BaseLLMProvider):
except NoSettingsContextError:
pass # Optional parameter
+ # Request usage stats on streamed responses (stream_options.
+ # include_usage). Opt-in via subclass kwargs because some
+ # OpenAI-compatible endpoints reject unknown request fields.
+ if kwargs.get("stream_usage") is not None:
+ llm_params["stream_usage"] = kwargs["stream_usage"]
+
logger.info(
f"Creating {cls.provider_name} LLM with model: {model_name}, "
f"temperature: {temperature}, endpoint: {base_url}"
diff --git a/src/local_deep_research/metrics/query_utils.py b/src/local_deep_research/metrics/query_utils.py
index 7b0e914be2..400dd179d8 100644
--- a/src/local_deep_research/metrics/query_utils.py
+++ b/src/local_deep_research/metrics/query_utils.py
@@ -7,17 +7,42 @@ from sqlalchemy import Column, case, func
from sqlalchemy.orm import Session
-PERIOD_DAYS_MAP = {"7d": 7, "30d": 30, "90d": 90, "365d": 365, "all": None}
+# Single source of truth for period → days across all metrics endpoints.
+# Covers BOTH UI vocabularies: the main dashboard / context-overflow pages
+# send '3m'/'1y'; the standalone link-analytics page sends '90d'/'365d'.
+# Both alias to the same day counts so every metrics endpoint agrees.
+# 'all' maps to None (no time limit).
+PERIOD_DAYS_MAP = {
+ "7d": 7,
+ "30d": 30,
+ "3m": 90,
+ "90d": 90,
+ "1y": 365,
+ "365d": 365,
+ "all": None,
+}
def get_period_days(period: str, default: int = 30) -> int | None:
"""Convert a period string to number of days.
- Returns None for 'all' (no time limit).
+ Returns None for 'all' (no time limit); unknown values use ``default``.
"""
return PERIOD_DAYS_MAP.get(period, default)
+def get_period_cutoff(period: str) -> datetime | None:
+ """UTC cutoff datetime for a period string, or None for 'all'.
+
+ Delegates to :func:`get_period_days` so there is exactly one period
+ vocabulary. Unknown values default to 30 days.
+ """
+ days = get_period_days(period)
+ if days is None: # 'all'
+ return None
+ return datetime.now(UTC) - timedelta(days=days)
+
+
def get_time_filter_condition(period: str, timestamp_column: Column) -> Any:
"""Get SQLAlchemy condition for time filtering.
@@ -28,20 +53,9 @@ def get_time_filter_condition(period: str, timestamp_column: Column) -> Any:
Returns:
SQLAlchemy condition object or None for 'all'
"""
- if period == "all":
+ cutoff = get_period_cutoff(period)
+ if cutoff is None:
return None
- if period == "7d":
- cutoff = datetime.now(UTC) - timedelta(days=7)
- elif period == "30d":
- cutoff = datetime.now(UTC) - timedelta(days=30)
- elif period == "3m":
- cutoff = datetime.now(UTC) - timedelta(days=90)
- elif period == "1y":
- cutoff = datetime.now(UTC) - timedelta(days=365)
- else:
- # Default to 30 days for unknown periods
- cutoff = datetime.now(UTC) - timedelta(days=30)
-
return timestamp_column >= cutoff
diff --git a/src/local_deep_research/metrics/token_counter.py b/src/local_deep_research/metrics/token_counter.py
index e40d18667b..d52686ee68 100644
--- a/src/local_deep_research/metrics/token_counter.py
+++ b/src/local_deep_research/metrics/token_counter.py
@@ -3,7 +3,7 @@
import inspect
import json
import time
-from datetime import datetime, timedelta, UTC
+from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Optional
@@ -13,7 +13,11 @@ from loguru import logger
from sqlalchemy import func, text
from ..database.models import ModelUsage, TokenUsage
-from .query_utils import get_research_mode_condition, get_time_filter_condition
+from .query_utils import (
+ get_period_cutoff,
+ get_research_mode_condition,
+ get_time_filter_condition,
+)
class TokenCountingCallback(BaseCallbackHandler):
@@ -59,6 +63,14 @@ class TokenCountingCallback(BaseCallbackHandler):
# Raw Ollama response metrics
self.ollama_metrics = {}
+ # Whether we've already logged the "provider reports no usage"
+ # warning. The callback is shared across every LLM call in a
+ # research session (see llm_config.py wrap_llm), so without this a
+ # 50-call run against a non-reporting provider logs 50 identical
+ # warnings. Calls are still recorded every time — only the warning
+ # is deduplicated.
+ self._warned_no_usage = False
+
# Track token counts in memory
self.counts = {
"total_tokens": 0,
@@ -506,6 +518,34 @@ class TokenCountingCallback(BaseCallbackHandler):
# Save to database if we have a research_id
if self.research_id:
self._save_to_db(prompt_tokens, completion_tokens)
+ elif self.research_id:
+ # The provider returned no usage data at all (e.g. OpenAI-
+ # compatible servers that omit `usage` on streamed responses,
+ # proxies that strip it, Ollama omitting prompt_eval_count for
+ # fully-cached prompts). Record the call anyway with zero token
+ # counts — model, provider, phase, response time and context-
+ # overflow estimates are still real data. Skipping the row
+ # entirely is what made every metrics page render as if LDR had
+ # never been used (#4457).
+ # Warn once per research session, not once per call.
+ if not self._warned_no_usage:
+ self._warned_no_usage = True
+ hint = ""
+ if self.current_provider == "openai_endpoint":
+ hint = (
+ " If your server supports stream_options.include_usage "
+ "(LM Studio 0.3.18+, llama.cpp, vLLM, OpenRouter), "
+ "enable the 'llm.openai_endpoint.stream_usage' setting "
+ "to get real token counts on streamed calls."
+ )
+ logger.warning(
+ f"LLM provider returned no token usage data - recording "
+ f"calls with zero counts. model={self.current_model} "
+ f"provider={self.current_provider} "
+ f"research_id={self.research_id}.{hint} "
+ f"(further occurrences this session are suppressed)"
+ )
+ self._save_to_db(0, 0)
def on_llm_error(self, error, **kwargs: Any) -> None:
"""Called when LLM encounters an error."""
@@ -914,26 +954,16 @@ class TokenCounter:
else:
logger.debug("No ResearchHistory records found in database")
- # Get time filter conditions for ResearchHistory query
- start_time, end_time = None, None
- if period != "all":
- if period == "today":
- start_time = datetime.now(UTC).replace(
- hour=0, minute=0, second=0, microsecond=0
- )
- elif period == "week":
- start_time = datetime.now(UTC) - timedelta(days=7)
- elif period == "month":
- start_time = datetime.now(UTC) - timedelta(days=30)
-
- if start_time:
- end_time = datetime.now(UTC)
-
- # Apply time filter if specified
- if start_time and end_time:
+ # Time filter for the ResearchHistory count. created_at is
+ # an ISO-8601 TEXT column, so compare isoformat strings.
+ # Must use the API's period vocabulary ('7d'/'30d'/'3m'/
+ # '1y'/'all') — the previous 'today'/'week'/'month' branches
+ # never matched it, so the research count silently ignored
+ # the selected period and always showed all-time.
+ start_time = get_period_cutoff(period)
+ if start_time is not None:
research_query = research_query.filter(
- ResearchHistory.created_at >= start_time.isoformat(),
- ResearchHistory.created_at <= end_time.isoformat(),
+ ResearchHistory.created_at >= start_time.isoformat()
)
# Apply mode filter if specified
diff --git a/tests/database/test_thread_metrics.py b/tests/database/test_thread_metrics.py
index 69824981b1..7e9057fee6 100644
--- a/tests/database/test_thread_metrics.py
+++ b/tests/database/test_thread_metrics.py
@@ -199,18 +199,21 @@ class TestWriteTokenMetrics:
"local_deep_research.database.models.TokenUsage",
mock_token_usage_class,
):
- token_data = {
- "model_name": "gpt-4",
- "provider": "openai",
- "prompt_tokens": 100,
- "completion_tokens": 50,
- }
-
- writer.write_token_metrics("testuser", 123, token_data)
-
- # Verify TokenUsage was created and added to session
- mock_token_usage_class.assert_called_once()
- mock_session.add.assert_called_once()
+ with patch("local_deep_research.database.models.ModelUsage"):
+ mock_session.query.return_value.filter_by.return_value.first.return_value = None
+
+ token_data = {
+ "model_name": "gpt-4",
+ "provider": "openai",
+ "prompt_tokens": 100,
+ "completion_tokens": 50,
+ }
+
+ writer.write_token_metrics("testuser", 123, token_data)
+
+ # Verify TokenUsage was created and added to session
+ mock_token_usage_class.assert_called_once()
+ assert mock_session.add.call_count == 2
class TestGlobalInstance:
diff --git a/tests/database/test_thread_metrics_writer.py b/tests/database/test_thread_metrics_writer.py
index 2fd64034ea..5aae5cfe19 100644
--- a/tests/database/test_thread_metrics_writer.py
+++ b/tests/database/test_thread_metrics_writer.py
@@ -474,6 +474,65 @@ class TestWriteTokenMetrics:
call_kwargs = mock_token_cls.call_args[1]
assert call_kwargs["research_id"] is None
+ # --- ModelUsage upsert -------------------------------------------------
+
+ def test_creates_model_usage_when_none_exists(
+ self, writer_with_password, mock_db_manager
+ ):
+ """New ModelUsage record is created when none exists."""
+ token_data = {
+ "model_name": "gpt-4",
+ "provider": "openai",
+ "prompt_tokens": 100,
+ "completion_tokens": 50,
+ }
+
+ mock_db, mock_session = mock_db_manager
+ mock_session.query.return_value.filter_by.return_value.first.return_value = None
+
+ with patch(
+ "local_deep_research.database.models.TokenUsage", MagicMock()
+ ):
+ with patch(
+ "local_deep_research.database.models.ModelUsage"
+ ) as mock_model_cls:
+ writer_with_password.write_token_metrics(
+ "testuser", 1, token_data
+ )
+
+ mock_model_cls.assert_called_once_with(
+ model_name="gpt-4",
+ model_provider="openai",
+ total_tokens=150,
+ total_calls=1,
+ )
+ assert mock_session.add.call_count == 2
+
+ def test_updates_existing_model_usage(
+ self, writer_with_password, mock_db_manager
+ ):
+ """Existing ModelUsage record is incremented."""
+ token_data = {
+ "model_name": "gpt-4",
+ "provider": "openai",
+ "prompt_tokens": 100,
+ "completion_tokens": 50,
+ }
+
+ mock_db, mock_session = mock_db_manager
+ existing = MagicMock()
+ existing.total_tokens = 1000
+ existing.total_calls = 5
+ mock_session.query.return_value.filter_by.return_value.first.return_value = existing
+
+ with patch(
+ "local_deep_research.database.models.TokenUsage", MagicMock()
+ ):
+ writer_with_password.write_token_metrics("testuser", 1, token_data)
+
+ assert existing.total_tokens == 1150
+ assert existing.total_calls == 6
+
# ===========================================================================
# Global instance
diff --git a/tests/llm_providers/test_custom_openai_endpoint.py b/tests/llm_providers/test_custom_openai_endpoint.py
index 77f17b09e5..eb8b3375bf 100644
--- a/tests/llm_providers/test_custom_openai_endpoint.py
+++ b/tests/llm_providers/test_custom_openai_endpoint.py
@@ -201,6 +201,63 @@ class TestCustomOpenAIEndpointCreateLLM:
call_kwargs = mock_chat_openai.call_args[1]
assert "172.19.0.5" in call_kwargs["base_url"]
+ @staticmethod
+ def _create_llm_with_settings(settings_map):
+ def side_effect(key, default=None, *args, **kwargs):
+ return settings_map.get(key, default)
+
+ with patch(
+ "local_deep_research.llm.providers.implementations.custom_openai_endpoint.get_setting_from_snapshot"
+ ) as mock_get_setting:
+ mock_get_setting.side_effect = side_effect
+
+ with patch(
+ "local_deep_research.config.thread_settings.get_setting_from_snapshot"
+ ) as mock_base_get_setting:
+ mock_base_get_setting.side_effect = side_effect
+
+ with patch(
+ "local_deep_research.llm.providers.openai_base.ChatOpenAI"
+ ) as mock_chat_openai:
+ mock_chat_openai.return_value = Mock()
+ CustomOpenAIEndpointProvider.create_llm(
+ model_name="test-model"
+ )
+ return mock_chat_openai.call_args[1]
+
+ def test_create_llm_stream_usage_enabled(self):
+ """stream_usage setting on -> passed to ChatOpenAI."""
+ call_kwargs = self._create_llm_with_settings(
+ {
+ "llm.openai_endpoint.url": "http://localhost:1234/v1",
+ "llm.openai_endpoint.api_key": "test-api-key",
+ "llm.openai_endpoint.stream_usage": True,
+ }
+ )
+ assert call_kwargs["stream_usage"] is True
+
+ def test_create_llm_stream_usage_off_by_default(self):
+ """Without the setting, stream_usage is not sent at all — some
+ OpenAI-compatible gateways 400 on unknown stream_options."""
+ call_kwargs = self._create_llm_with_settings(
+ {
+ "llm.openai_endpoint.url": "http://localhost:1234/v1",
+ "llm.openai_endpoint.api_key": "test-api-key",
+ }
+ )
+ assert "stream_usage" not in call_kwargs
+
+ def test_create_llm_stream_usage_explicit_false_not_sent(self):
+ """An explicit false behaves like the default: parameter omitted."""
+ call_kwargs = self._create_llm_with_settings(
+ {
+ "llm.openai_endpoint.url": "http://localhost:1234/v1",
+ "llm.openai_endpoint.api_key": "test-api-key",
+ "llm.openai_endpoint.stream_usage": False,
+ }
+ )
+ assert "stream_usage" not in call_kwargs
+
class TestCustomOpenAIEndpointAPISignature:
"""Tests for API method signatures.
diff --git a/tests/metrics/test_query_utils.py b/tests/metrics/test_query_utils.py
index 552a5614b9..f26b1f063b 100644
--- a/tests/metrics/test_query_utils.py
+++ b/tests/metrics/test_query_utils.py
@@ -3,7 +3,11 @@
from sqlalchemy import Column, DateTime, String
from sqlalchemy.orm import declarative_base
+from datetime import datetime, timedelta, UTC
+
from local_deep_research.metrics.query_utils import (
+ get_period_cutoff,
+ get_period_days,
get_research_mode_condition,
get_time_filter_condition,
)
@@ -21,6 +25,56 @@ class QueryTestModel(Base):
mode = Column(String)
+class TestGetPeriodDays:
+ """Tests for get_period_days — single period vocabulary (both UIs)."""
+
+ def test_all_returns_none(self):
+ assert get_period_days("all") is None
+
+ def test_dashboard_vocabulary(self):
+ # Main dashboard / context-overflow buttons send these.
+ assert get_period_days("7d") == 7
+ assert get_period_days("30d") == 30
+ assert get_period_days("3m") == 90
+ assert get_period_days("1y") == 365
+
+ def test_link_analytics_vocabulary(self):
+ # The standalone link-analytics page sends these aliases; they must
+ # resolve to the same day counts as 3m/1y.
+ assert get_period_days("90d") == 90
+ assert get_period_days("365d") == 365
+
+ def test_unknown_uses_default(self):
+ assert get_period_days("nonsense") == 30
+ assert get_period_days("nonsense", default=7) == 7
+
+
+class TestGetPeriodCutoff:
+ """Tests for get_period_cutoff function."""
+
+ def test_returns_none_for_all(self):
+ assert get_period_cutoff("all") is None
+
+ def test_known_periods_map_to_days(self):
+ now = datetime.now(UTC)
+ for period, days in [
+ ("7d", 7),
+ ("30d", 30),
+ ("3m", 90),
+ ("90d", 90),
+ ("1y", 365),
+ ("365d", 365),
+ ]:
+ cutoff = get_period_cutoff(period)
+ expected = now - timedelta(days=days)
+ assert abs((cutoff - expected).total_seconds()) < 5, period
+
+ def test_unknown_period_defaults_to_30_days(self):
+ cutoff = get_period_cutoff("month") # old vocabulary, no longer valid
+ expected = datetime.now(UTC) - timedelta(days=30)
+ assert abs((cutoff - expected).total_seconds()) < 5
+
+
class TestGetTimeFilterCondition:
"""Tests for get_time_filter_condition function."""
diff --git a/tests/metrics/test_token_counter_callback.py b/tests/metrics/test_token_counter_callback.py
index 8f9de6b943..91ad82ab34 100644
--- a/tests/metrics/test_token_counter_callback.py
+++ b/tests/metrics/test_token_counter_callback.py
@@ -2,7 +2,7 @@
model detection, provider detection, context overflow, and error handling."""
import time
-from unittest.mock import Mock
+from unittest.mock import Mock, patch
from langchain_core.outputs import LLMResult
@@ -404,6 +404,90 @@ class TestTokenCountingCallbackContextOverflow:
assert callback.context_truncated is False
+class TestTokenCountingCallbackMissingUsageData:
+ """Tests for recording calls when the provider reports no usage (#4457)."""
+
+ @staticmethod
+ def _response_without_usage():
+ response = Mock(spec=LLMResult)
+ response.llm_output = None
+ response.generations = []
+ return response
+
+ def test_no_usage_with_research_id_still_saves_to_db(self):
+ """A call without usage data must still be recorded (zero counts)."""
+ callback = TokenCountingCallback(research_id="research-123")
+ callback.current_model = "test-model"
+ callback._save_to_db = Mock()
+
+ callback.on_llm_end(self._response_without_usage())
+
+ callback._save_to_db.assert_called_once_with(0, 0)
+
+ def test_no_usage_without_research_id_does_not_save(self):
+ """Without a research_id there is nothing to persist."""
+ callback = TokenCountingCallback()
+ callback.current_model = "test-model"
+ callback._save_to_db = Mock()
+
+ callback.on_llm_end(self._response_without_usage())
+
+ callback._save_to_db.assert_not_called()
+
+ def test_no_usage_does_not_update_counts(self):
+ """Zero-count recording must not inflate in-memory token counts."""
+ callback = TokenCountingCallback(research_id="research-123")
+ callback.current_model = "test-model"
+ callback._save_to_db = Mock()
+
+ callback.on_llm_end(self._response_without_usage())
+
+ assert callback.counts["total_tokens"] == 0
+ assert callback.counts["total_prompt_tokens"] == 0
+ assert callback.counts["total_completion_tokens"] == 0
+
+ def test_repeated_no_usage_records_every_call_warns_once(self):
+ """Every no-usage call is recorded, but the warning fires once."""
+ import local_deep_research.metrics.token_counter as tc_mod
+
+ callback = TokenCountingCallback(research_id="research-123")
+ callback.current_model = "test-model"
+ callback._save_to_db = Mock()
+
+ with patch.object(tc_mod.logger, "warning") as mock_warn:
+ for _ in range(5):
+ callback.on_llm_end(self._response_without_usage())
+
+ assert callback._save_to_db.call_count == 5
+ mock_warn.assert_called_once()
+
+ def test_usage_present_saves_actual_counts(self):
+ """Sanity check: usage data still saves the real counts."""
+ callback = TokenCountingCallback(research_id="research-123")
+ callback.current_model = "gpt-4"
+ callback.counts["by_model"]["gpt-4"] = {
+ "prompt_tokens": 0,
+ "completion_tokens": 0,
+ "total_tokens": 0,
+ "calls": 1,
+ "provider": "openai",
+ }
+ callback._save_to_db = Mock()
+
+ response = Mock(spec=LLMResult)
+ response.llm_output = {
+ "token_usage": {
+ "prompt_tokens": 10,
+ "completion_tokens": 5,
+ "total_tokens": 15,
+ }
+ }
+ response.generations = []
+ callback.on_llm_end(response)
+
+ callback._save_to_db.assert_called_once_with(10, 5)
+
+
class TestTokenCountingCallbackErrorHandling:
"""Tests for error tracking in callback."""
diff --git a/tests/metrics/test_token_counter_deep_coverage.py b/tests/metrics/test_token_counter_deep_coverage.py
index 39e556e170..60f5a83179 100644
--- a/tests/metrics/test_token_counter_deep_coverage.py
+++ b/tests/metrics/test_token_counter_deep_coverage.py
@@ -247,11 +247,12 @@ class TestOnLlmEndTokenUsage:
assert cb.counts["total_tokens"] == 60
save_mock.assert_called_once_with(40, 20)
- def test_no_token_usage_no_db_save(self):
+ def test_no_token_usage_saves_zero_counts(self):
+ """No usage data from the provider still records the call (#4457)."""
cb = _make_callback()
result = _make_llm_result(llm_output=None, generations=[])
save_mock = self._run_end(cb, result)
- save_mock.assert_not_called()
+ save_mock.assert_called_once_with(0, 0)
def test_response_time_calculated(self):
cb = _make_callback()
diff --git a/tests/settings/golden_master_settings.json b/tests/settings/golden_master_settings.json
index 94b8dfcd63..deae0f6d0b 100644
--- a/tests/settings/golden_master_settings.json
+++ b/tests/settings/golden_master_settings.json
@@ -1387,6 +1387,20 @@
"value": "",
"visible": true
},
+ "llm.openai_endpoint.stream_usage": {
+ "category": null,
+ "description": "Request token usage statistics on streamed responses (sends stream_options.include_usage). Enable this if the token metrics dashboard shows zero token counts for streamed calls. Supported by LM Studio (0.3.18+), llama.cpp server, vLLM, and OpenRouter. Leave disabled if your endpoint rejects requests with a 'stream_options' error (e.g. some API gateways).",
+ "editable": true,
+ "max_value": null,
+ "min_value": null,
+ "name": "Stream Token Usage",
+ "options": null,
+ "step": null,
+ "type": "LLM",
+ "ui_element": "checkbox",
+ "value": false,
+ "visible": true
+ },
"llm.openai_endpoint.url": {
"category": null,
"description": "URL of a custom OpenAI-compatible API endpoint. Use for llama.cpp server, vLLM, Ollama, OpenRouter, Groq, Together, or any service that implements the OpenAI API format. Example for local llama.cpp: http://localhost:8000/v1",