Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(metrics): keep raw Ollama metrics with each response (#7286)

LearningCircuit1bc91bcb13d3
diff --git a/changelog.d/+ollama-call-metrics.bugfix.md b/changelog.d/+ollama-call-metrics.bugfix.md
new file mode 100644
index 0000000000..d3c140b8c1
--- /dev/null
+++ b/changelog.d/+ollama-call-metrics.bugfix.md
@@ -0,0 +1 @@
+Keep raw Ollama token counters and durations attached to their own response, so later or overlapping LLM calls cannot inherit another call's metrics.
diff --git a/src/local_deep_research/metrics/token_counter.py b/src/local_deep_research/metrics/token_counter.py
index 30fb8ea98d..e7b7c7d707 100644
--- a/src/local_deep_research/metrics/token_counter.py
+++ b/src/local_deep_research/metrics/token_counter.py
@@ -419,6 +419,7 @@ class TokenCountingCallback(BaseCallbackHandler):
# Extract token usage from response
token_usage = None
+ ollama_metrics = {}
# Check multiple locations for token usage
if hasattr(response, "llm_output") and response.llm_output:
@@ -476,7 +477,7 @@ class TokenCountingCallback(BaseCallbackHandler):
"eval_count"
):
# Capture raw Ollama metrics
- self.ollama_metrics = {
+ ollama_metrics = {
"prompt_eval_count": resp_meta.get(
"prompt_eval_count"
),
@@ -519,6 +520,12 @@ class TokenCountingCallback(BaseCallbackHandler):
if token_usage:
break
+ # Persist the metrics from this response, even when another call
+ # finishes before its database write. Keep the attribute as a
+ # compatibility mirror, not the source for this call's saved row.
+ call_meta["ollama_metrics"] = ollama_metrics
+ self.ollama_metrics = ollama_metrics
+
# Estimation-based overflow detection for providers that don't echo
# prompt_eval_count (OpenAI, Anthropic, OpenRouter, etc.). The Ollama
# path above sets context_truncated=True if it detected provider-
@@ -656,6 +663,8 @@ class TokenCountingCallback(BaseCallbackHandler):
)
# Still save to database to track failed calls
+ call_meta["ollama_metrics"] = {}
+ self.ollama_metrics = {}
if self.research_id:
self._save_to_db(0, 0, call_meta)
@@ -705,8 +714,12 @@ class TokenCountingCallback(BaseCallbackHandler):
"call_stack": self.call_stack,
}
- def _get_context_overflow_fields(self) -> Dict[str, Any]:
+ def _get_context_overflow_fields(
+ self, ollama_metrics: Optional[Dict[str, Any]] = None
+ ) -> Dict[str, Any]:
"""Get context overflow detection fields for database saving."""
+ if ollama_metrics is None:
+ ollama_metrics = self.ollama_metrics
return {
"context_limit": self.context_limit,
"context_truncated": self.context_truncated, # Now Boolean
@@ -717,16 +730,14 @@ class TokenCountingCallback(BaseCallbackHandler):
if self.context_truncated
else None,
# Raw Ollama metrics
- "ollama_prompt_eval_count": self.ollama_metrics.get(
- "prompt_eval_count"
- ),
- "ollama_eval_count": self.ollama_metrics.get("eval_count"),
- "ollama_total_duration": self.ollama_metrics.get("total_duration"),
- "ollama_load_duration": self.ollama_metrics.get("load_duration"),
- "ollama_prompt_eval_duration": self.ollama_metrics.get(
+ "ollama_prompt_eval_count": ollama_metrics.get("prompt_eval_count"),
+ "ollama_eval_count": ollama_metrics.get("eval_count"),
+ "ollama_total_duration": ollama_metrics.get("total_duration"),
+ "ollama_load_duration": ollama_metrics.get("load_duration"),
+ "ollama_prompt_eval_duration": ollama_metrics.get(
"prompt_eval_duration"
),
- "ollama_eval_duration": self.ollama_metrics.get("eval_duration"),
+ "ollama_eval_duration": ollama_metrics.get("eval_duration"),
}
def _save_to_db(
@@ -745,7 +756,8 @@ class TokenCountingCallback(BaseCallbackHandler):
the caller. Defaults to the sum of the two counts above
when the caller has no provider-reported total.
call_meta: Per-run call metadata captured by on_llm_start
- (calling_file / calling_function / call_stack). When the
+ (calling_file / calling_function / call_stack), plus the
+ raw Ollama metrics from on_llm_end or on_llm_error. When the
callback is invoked outside on_llm_end / on_llm_error
(e.g. legacy tests that set the attributes directly),
``call_meta`` may be ``None`` and the instance attributes
@@ -763,6 +775,9 @@ class TokenCountingCallback(BaseCallbackHandler):
# Check if we're in a thread - if so, queue the save for later
import threading
+ overflow_fields = self._get_context_overflow_fields(
+ call_meta.get("ollama_metrics")
+ )
if threading.current_thread().name != "MainThread":
# Use thread-safe metrics database for background threads
username = (
@@ -831,7 +846,7 @@ class TokenCountingCallback(BaseCallbackHandler):
"calling_function": call_meta.get("calling_function"),
"call_stack": call_meta.get("call_stack"),
# Add context overflow fields using helper method
- **self._get_context_overflow_fields(),
+ **overflow_fields,
}
# Convert list to JSON string if needed
@@ -922,7 +937,7 @@ class TokenCountingCallback(BaseCallbackHandler):
f"Saving TokenUsage - context_limit: {self.context_limit}, "
f"context_truncated: {self.context_truncated}, "
f"tokens_truncated: {self.tokens_truncated}, "
- f"ollama_prompt_eval_count: {self.ollama_metrics.get('prompt_eval_count')}, "
+ f"ollama_prompt_eval_count: {overflow_fields['ollama_prompt_eval_count']}, "
f"prompt_tokens: {prompt_tokens}, "
f"completion_tokens: {completion_tokens}"
)
@@ -953,7 +968,7 @@ class TokenCountingCallback(BaseCallbackHandler):
calling_function=call_meta.get("calling_function"),
call_stack=call_meta.get("call_stack"),
# Add context overflow fields using helper method
- **self._get_context_overflow_fields(),
+ **overflow_fields,
)
session.add(token_usage)
diff --git a/tests/metrics/test_token_counting_attribution.py b/tests/metrics/test_token_counting_attribution.py
index f0e9e0c02a..83892eb439 100644
--- a/tests/metrics/test_token_counting_attribution.py
+++ b/tests/metrics/test_token_counting_attribution.py
@@ -719,16 +719,6 @@ def test_success_status_does_not_stick_after_an_earlier_failure(user_dbs):
assert succeeded.error_type is None
- strict=True,
- reason=(
- "RAW OLLAMA COUNTERS BLEED ACROSS CALLS. self.ollama_metrics is set "
- "in on_llm_end for Ollama responses and never cleared by "
- "on_llm_start, so the next call -- from any provider -- persists the "
- "previous Ollama call's prompt_eval_count / eval_count / durations "
- "on its own row via _get_context_overflow_fields()."
- ),
-)
def test_ollama_raw_metrics_do_not_bleed_into_the_next_providers_row(user_dbs):
callback = make_callback("research-alice", "alice")
@@ -758,6 +748,127 @@ def test_ollama_raw_metrics_do_not_bleed_into_the_next_providers_row(user_dbs):
assert openai_row.ollama_total_duration is None
+def raw_ollama_values(row):
+ return (
+ row.ollama_prompt_eval_count,
+ row.ollama_eval_count,
+ row.ollama_total_duration,
+ row.ollama_load_duration,
+ row.ollama_prompt_eval_duration,
+ row.ollama_eval_duration,
+ )
+
+
[email protected]("background", [False, True])
[email protected]("registered", [False, True])
+ "second_kind", ["token_usage", "usage_metadata", "missing", "error", "raw"]
+)
+def test_raw_ollama_metrics_belong_only_to_the_response_that_supplied_them(
+ user_dbs, background, registered, second_kind
+):
+ callback = make_callback("research-alice", "alice")
+ callback.preset_model, callback.preset_provider = "llama3", "ollama"
+
+ def record_calls():
+ with request_user("alice"):
+ callback.on_llm_start({"_type": "ChatOllama"}, ["p"], run_id="r1")
+ callback.on_llm_end(StubOllamaResult(900, 100, 555), run_id="r1")
+ kwargs = {"run_id": "r2"} if registered else {}
+ if registered:
+ callback.on_llm_start({"_type": "ChatOllama"}, ["p"], **kwargs)
+
+ if second_kind == "error":
+ callback.on_llm_error(
+ TimeoutError("upstream timeout"), **kwargs
+ )
+ return
+ if second_kind == "token_usage":
+ response = StubLLMResult(10, 4)
+ elif second_kind == "raw":
+ response = StubOllamaResult(0, 2, 0)
+ else:
+ response = StubOllamaResult(0, 0, 0)
+ message = response.generations[0][0].message
+ message.response_metadata = {}
+ if second_kind == "usage_metadata":
+ message.usage_metadata = {
+ "input_tokens": 10,
+ "output_tokens": 4,
+ "total_tokens": 14,
+ }
+ callback.on_llm_end(response, **kwargs)
+
+ if background:
+ with ThreadPoolExecutor(max_workers=1) as pool:
+ pool.submit(record_calls).result(timeout=10)
+ else:
+ record_calls()
+
+ first, second = user_dbs.rows("alice", TokenUsage)
+ assert raw_ollama_values(first) == (900, 100, 555, 1_000, 2_000, 3_000)
+ if second_kind == "raw":
+ assert raw_ollama_values(second) == (0, 2, 0, 1_000, 2_000, 3_000)
+ assert second.total_tokens == 2
+ else:
+ assert raw_ollama_values(second) == (None,) * 6
+ assert second.total_tokens == (
+ 14 if second_kind in {"token_usage", "usage_metadata"} else 0
+ )
+ if second_kind == "error":
+ assert second.success_status == "error"
+ assert second.error_type == "TimeoutError"
+
+
[email protected]("second_has_raw_metrics", [False, True])
+def test_overlapping_completions_keep_their_own_raw_metrics(
+ user_dbs, monkeypatch, second_has_raw_metrics
+):
+ """Finish a second response while the first waits to write its row."""
+ callback = make_callback("research-alice", "alice")
+ callback.preset_model, callback.preset_provider = "llama3", "ollama"
+ callback.on_llm_start({"_type": "ChatOllama"}, ["p"], run_id="r1")
+ callback.on_llm_start({"_type": "ChatOllama"}, ["p"], run_id="r2")
+ user_dbs.engine("alice")
+ first_waiting = threading.Event()
+ release_first = threading.Event()
+ save = callback._save_to_db
+
+ def pause_first_save(prompt_tokens, completion_tokens, *args, **kwargs):
+ if prompt_tokens == 900:
+ first_waiting.set()
+ assert release_first.wait(timeout=10)
+ return save(prompt_tokens, completion_tokens, *args, **kwargs)
+
+ monkeypatch.setattr(callback, "_save_to_db", pause_first_save)
+ second_response = (
+ StubOllamaResult(10, 4, 777)
+ if second_has_raw_metrics
+ else StubLLMResult(10, 4)
+ )
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ first = pool.submit(
+ callback.on_llm_end, StubOllamaResult(900, 100, 555), run_id="r1"
+ )
+ try:
+ assert first_waiting.wait(timeout=10)
+ pool.submit(
+ callback.on_llm_end, second_response, run_id="r2"
+ ).result(timeout=10)
+ finally:
+ release_first.set()
+ first.result(timeout=10)
+
+ rows = {r.prompt_tokens: r for r in user_dbs.rows("alice", TokenUsage)}
+ assert set(rows) == {900, 10}
+ assert raw_ollama_values(rows[900]) == (900, 100, 555, 1_000, 2_000, 3_000)
+ assert raw_ollama_values(rows[10]) == (
+ (10, 4, 777, 1_000, 2_000, 3_000)
+ if second_has_raw_metrics
+ else (None,) * 6
+ )
+
+
def test_retry_attempts_are_counted_once_each_not_double_incremented(user_dbs):
"""One row and one call-count increment per real attempt.
diff --git a/tests/test_token_counter_coverage.py b/tests/test_token_counter_coverage.py
index 3535c6ee0f..eab2c23cde 100644
--- a/tests/test_token_counter_coverage.py
+++ b/tests/test_token_counter_coverage.py
@@ -457,6 +457,7 @@ class TestOnLlmEnd:
"calling_file": None,
"calling_function": None,
"call_stack": None,
+ "ollama_metrics": {},
},
total_tokens=30,
)
@@ -637,6 +638,7 @@ class TestOnLlmError:
"calling_file": None,
"calling_function": None,
"call_stack": None,
+ "ollama_metrics": {},
},
)