~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(metrics): keep raw Ollama metrics with each response (#7286)
LearningCircuit1bc91bcb13d3
diff --git a/changelog.d/+ollama-call-metrics.bugfix.md b/changelog.d/+ollama-call-metrics.bugfix.mdnew file mode 100644index 0000000000..d3c140b8c1--- /dev/null+++ b/changelog.d/+ollama-call-metrics.bugfix.md@@ -0,0 +1 @@+Keep raw Ollama token counters and durations attached to their own response, so later or overlapping LLM calls cannot inherit another call's metrics.diff --git a/src/local_deep_research/metrics/token_counter.py b/src/local_deep_research/metrics/token_counter.pyindex 30fb8ea98d..e7b7c7d707 100644--- a/src/local_deep_research/metrics/token_counter.py+++ b/src/local_deep_research/metrics/token_counter.py@@ -419,6 +419,7 @@ class TokenCountingCallback(BaseCallbackHandler):# Extract token usage from responsetoken_usage = None+ ollama_metrics = {}# Check multiple locations for token usageif hasattr(response, "llm_output") and response.llm_output:@@ -476,7 +477,7 @@ class TokenCountingCallback(BaseCallbackHandler):"eval_count"):# Capture raw Ollama metrics- self.ollama_metrics = {+ ollama_metrics = {"prompt_eval_count": resp_meta.get("prompt_eval_count"),@@ -519,6 +520,12 @@ class TokenCountingCallback(BaseCallbackHandler):if token_usage:break+ # Persist the metrics from this response, even when another call+ # finishes before its database write. Keep the attribute as a+ # compatibility mirror, not the source for this call's saved row.+ call_meta["ollama_metrics"] = ollama_metrics+ self.ollama_metrics = ollama_metrics+# Estimation-based overflow detection for providers that don't echo# prompt_eval_count (OpenAI, Anthropic, OpenRouter, etc.). The Ollama# path above sets context_truncated=True if it detected provider-@@ -656,6 +663,8 @@ class TokenCountingCallback(BaseCallbackHandler):)# Still save to database to track failed calls+ call_meta["ollama_metrics"] = {}+ self.ollama_metrics = {}if self.research_id:self._save_to_db(0, 0, call_meta)@@ -705,8 +714,12 @@ class TokenCountingCallback(BaseCallbackHandler):"call_stack": self.call_stack,}- def _get_context_overflow_fields(self) -> Dict[str, Any]:+ def _get_context_overflow_fields(+ self, ollama_metrics: Optional[Dict[str, Any]] = None+ ) -> Dict[str, Any]:"""Get context overflow detection fields for database saving."""+ if ollama_metrics is None:+ ollama_metrics = self.ollama_metricsreturn {"context_limit": self.context_limit,"context_truncated": self.context_truncated, # Now Boolean@@ -717,16 +730,14 @@ class TokenCountingCallback(BaseCallbackHandler):if self.context_truncatedelse None,# Raw Ollama metrics- "ollama_prompt_eval_count": self.ollama_metrics.get(- "prompt_eval_count"- ),- "ollama_eval_count": self.ollama_metrics.get("eval_count"),- "ollama_total_duration": self.ollama_metrics.get("total_duration"),- "ollama_load_duration": self.ollama_metrics.get("load_duration"),- "ollama_prompt_eval_duration": self.ollama_metrics.get(+ "ollama_prompt_eval_count": ollama_metrics.get("prompt_eval_count"),+ "ollama_eval_count": ollama_metrics.get("eval_count"),+ "ollama_total_duration": ollama_metrics.get("total_duration"),+ "ollama_load_duration": ollama_metrics.get("load_duration"),+ "ollama_prompt_eval_duration": ollama_metrics.get("prompt_eval_duration"),- "ollama_eval_duration": self.ollama_metrics.get("eval_duration"),+ "ollama_eval_duration": ollama_metrics.get("eval_duration"),}def _save_to_db(@@ -745,7 +756,8 @@ class TokenCountingCallback(BaseCallbackHandler):the caller. Defaults to the sum of the two counts abovewhen the caller has no provider-reported total.call_meta: Per-run call metadata captured by on_llm_start- (calling_file / calling_function / call_stack). When the+ (calling_file / calling_function / call_stack), plus the+ raw Ollama metrics from on_llm_end or on_llm_error. When thecallback is invoked outside on_llm_end / on_llm_error(e.g. legacy tests that set the attributes directly),``call_meta`` may be ``None`` and the instance attributes@@ -763,6 +775,9 @@ class TokenCountingCallback(BaseCallbackHandler):# Check if we're in a thread - if so, queue the save for laterimport threading+ overflow_fields = self._get_context_overflow_fields(+ call_meta.get("ollama_metrics")+ )if threading.current_thread().name != "MainThread":# Use thread-safe metrics database for background threadsusername = (@@ -831,7 +846,7 @@ class TokenCountingCallback(BaseCallbackHandler):"calling_function": call_meta.get("calling_function"),"call_stack": call_meta.get("call_stack"),# Add context overflow fields using helper method- **self._get_context_overflow_fields(),+ **overflow_fields,}# Convert list to JSON string if needed@@ -922,7 +937,7 @@ class TokenCountingCallback(BaseCallbackHandler):f"Saving TokenUsage - context_limit: {self.context_limit}, "f"context_truncated: {self.context_truncated}, "f"tokens_truncated: {self.tokens_truncated}, "- f"ollama_prompt_eval_count: {self.ollama_metrics.get('prompt_eval_count')}, "+ f"ollama_prompt_eval_count: {overflow_fields['ollama_prompt_eval_count']}, "f"prompt_tokens: {prompt_tokens}, "f"completion_tokens: {completion_tokens}")@@ -953,7 +968,7 @@ class TokenCountingCallback(BaseCallbackHandler):calling_function=call_meta.get("calling_function"),call_stack=call_meta.get("call_stack"),# Add context overflow fields using helper method- **self._get_context_overflow_fields(),+ **overflow_fields,)session.add(token_usage)diff --git a/tests/metrics/test_token_counting_attribution.py b/tests/metrics/test_token_counting_attribution.pyindex f0e9e0c02a..83892eb439 100644--- a/tests/metrics/test_token_counting_attribution.py+++ b/tests/metrics/test_token_counting_attribution.py@@ -719,16 +719,6 @@ def test_success_status_does_not_stick_after_an_earlier_failure(user_dbs):assert succeeded.error_type is None- strict=True,- reason=(- "RAW OLLAMA COUNTERS BLEED ACROSS CALLS. self.ollama_metrics is set "- "in on_llm_end for Ollama responses and never cleared by "- "on_llm_start, so the next call -- from any provider -- persists the "- "previous Ollama call's prompt_eval_count / eval_count / durations "- "on its own row via _get_context_overflow_fields()."- ),-)def test_ollama_raw_metrics_do_not_bleed_into_the_next_providers_row(user_dbs):callback = make_callback("research-alice", "alice")@@ -758,6 +748,127 @@ def test_ollama_raw_metrics_do_not_bleed_into_the_next_providers_row(user_dbs):assert openai_row.ollama_total_duration is None+def raw_ollama_values(row):+ return (+ row.ollama_prompt_eval_count,+ row.ollama_eval_count,+ row.ollama_total_duration,+ row.ollama_load_duration,+ row.ollama_prompt_eval_duration,+ row.ollama_eval_duration,+ )++[email protected]("background", [False, True])[email protected]("registered", [False, True])+ "second_kind", ["token_usage", "usage_metadata", "missing", "error", "raw"]+)+def test_raw_ollama_metrics_belong_only_to_the_response_that_supplied_them(+ user_dbs, background, registered, second_kind+):+ callback = make_callback("research-alice", "alice")+ callback.preset_model, callback.preset_provider = "llama3", "ollama"++ def record_calls():+ with request_user("alice"):+ callback.on_llm_start({"_type": "ChatOllama"}, ["p"], run_id="r1")+ callback.on_llm_end(StubOllamaResult(900, 100, 555), run_id="r1")+ kwargs = {"run_id": "r2"} if registered else {}+ if registered:+ callback.on_llm_start({"_type": "ChatOllama"}, ["p"], **kwargs)++ if second_kind == "error":+ callback.on_llm_error(+ TimeoutError("upstream timeout"), **kwargs+ )+ return+ if second_kind == "token_usage":+ response = StubLLMResult(10, 4)+ elif second_kind == "raw":+ response = StubOllamaResult(0, 2, 0)+ else:+ response = StubOllamaResult(0, 0, 0)+ message = response.generations[0][0].message+ message.response_metadata = {}+ if second_kind == "usage_metadata":+ message.usage_metadata = {+ "input_tokens": 10,+ "output_tokens": 4,+ "total_tokens": 14,+ }+ callback.on_llm_end(response, **kwargs)++ if background:+ with ThreadPoolExecutor(max_workers=1) as pool:+ pool.submit(record_calls).result(timeout=10)+ else:+ record_calls()++ first, second = user_dbs.rows("alice", TokenUsage)+ assert raw_ollama_values(first) == (900, 100, 555, 1_000, 2_000, 3_000)+ if second_kind == "raw":+ assert raw_ollama_values(second) == (0, 2, 0, 1_000, 2_000, 3_000)+ assert second.total_tokens == 2+ else:+ assert raw_ollama_values(second) == (None,) * 6+ assert second.total_tokens == (+ 14 if second_kind in {"token_usage", "usage_metadata"} else 0+ )+ if second_kind == "error":+ assert second.success_status == "error"+ assert second.error_type == "TimeoutError"++[email protected]("second_has_raw_metrics", [False, True])+def test_overlapping_completions_keep_their_own_raw_metrics(+ user_dbs, monkeypatch, second_has_raw_metrics+):+ """Finish a second response while the first waits to write its row."""+ callback = make_callback("research-alice", "alice")+ callback.preset_model, callback.preset_provider = "llama3", "ollama"+ callback.on_llm_start({"_type": "ChatOllama"}, ["p"], run_id="r1")+ callback.on_llm_start({"_type": "ChatOllama"}, ["p"], run_id="r2")+ user_dbs.engine("alice")+ first_waiting = threading.Event()+ release_first = threading.Event()+ save = callback._save_to_db++ def pause_first_save(prompt_tokens, completion_tokens, *args, **kwargs):+ if prompt_tokens == 900:+ first_waiting.set()+ assert release_first.wait(timeout=10)+ return save(prompt_tokens, completion_tokens, *args, **kwargs)++ monkeypatch.setattr(callback, "_save_to_db", pause_first_save)+ second_response = (+ StubOllamaResult(10, 4, 777)+ if second_has_raw_metrics+ else StubLLMResult(10, 4)+ )+ with ThreadPoolExecutor(max_workers=2) as pool:+ first = pool.submit(+ callback.on_llm_end, StubOllamaResult(900, 100, 555), run_id="r1"+ )+ try:+ assert first_waiting.wait(timeout=10)+ pool.submit(+ callback.on_llm_end, second_response, run_id="r2"+ ).result(timeout=10)+ finally:+ release_first.set()+ first.result(timeout=10)++ rows = {r.prompt_tokens: r for r in user_dbs.rows("alice", TokenUsage)}+ assert set(rows) == {900, 10}+ assert raw_ollama_values(rows[900]) == (900, 100, 555, 1_000, 2_000, 3_000)+ assert raw_ollama_values(rows[10]) == (+ (10, 4, 777, 1_000, 2_000, 3_000)+ if second_has_raw_metrics+ else (None,) * 6+ )++def test_retry_attempts_are_counted_once_each_not_double_incremented(user_dbs):"""One row and one call-count increment per real attempt.diff --git a/tests/test_token_counter_coverage.py b/tests/test_token_counter_coverage.pyindex 3535c6ee0f..eab2c23cde 100644--- a/tests/test_token_counter_coverage.py+++ b/tests/test_token_counter_coverage.py@@ -457,6 +457,7 @@ class TestOnLlmEnd:"calling_file": None,"calling_function": None,"call_stack": None,+ "ollama_metrics": {},},total_tokens=30,)@@ -637,6 +638,7 @@ class TestOnLlmError:"calling_file": None,"calling_function": None,"call_stack": None,+ "ollama_metrics": {},},)