~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
feat(metrics): structured truncation log + estimation-based detection (#3791)
LearningCircuit4c1c8d33f23f
diff --git a/changelog.d/+context-overflow-log-fields.feature.md b/changelog.d/+context-overflow-log-fields.feature.mdnew file mode 100644index 0000000000..23ad101548--- /dev/null+++ b/changelog.d/+context-overflow-log-fields.feature.md@@ -0,0 +1 @@+Context-overflow warnings now include structured fields (`research_id`, `model`, `provider`) and detect both input-only and total-context (input+output) overflow. Four distinct tags (`[provider-confirmed]`, `[total-context]`, `[estimated]`, `[estimated-total-context]`) make different overflow causes distinguishable in logs and alerts. Hosted providers (OpenAI, Anthropic, OpenRouter) get estimation-based detection so their truncation events are no longer invisible. The high-context VRAM warning now links directly to the context-overflow metrics page.diff --git a/src/local_deep_research/metrics/token_counter.py b/src/local_deep_research/metrics/token_counter.pyindex 29d3f41515..e40d18667b 100644--- a/src/local_deep_research/metrics/token_counter.py+++ b/src/local_deep_research/metrics/token_counter.py@@ -74,6 +74,16 @@ class TokenCountingCallback(BaseCallbackHandler):# Phase 1 Enhancement: Start timingself.start_time = time.time()+ # Reset per-call truncation state. The callback instance is shared+ # across every LLM call in a research session (see llm_config.py+ # wrap_llm), so without this reset the post-loop estimation block's+ # `if not self.context_truncated` guard would silently disable+ # [estimated] / [estimated-total-context] detection on every call+ # after the first one that truncates.+ self.context_truncated = False+ self.tokens_truncated = 0+ self.truncation_ratio = 0.0+# Estimate original prompt size (rough estimate: ~4 chars per token)if prompts:total_chars = sum(len(prompt) for prompt in prompts)@@ -221,31 +231,35 @@ class TokenCountingCallback(BaseCallbackHandler):self.counts["by_model"][self.current_model]["calls"] += 1def _check_context_overflow(- self, prompt_eval_count: int, source: str = ""+ self,+ prompt_eval_count: int,+ completion_tokens: int = 0,+ source: str = "",) -> None:- """Check for context overflow based on prompt token usage.+ """Check for context overflow based on prompt and total token usage.Args:prompt_eval_count: Number of tokens the model actually processed.+ completion_tokens: Number of tokens generated (for total-context check).source: Which branch provided the data (for logging)."""logger.debug(f"Context overflow check [{source}]: "f"prompt_eval_count={prompt_eval_count}, "+ f"completion_tokens={completion_tokens}, "f"context_limit={self.context_limit}")if not self.context_limit or prompt_eval_count <= 0:return- # Flag when utilization >= 80% or hard overflow+ # Input-only overflow: prompt at >= 80% of context limit. Matches the+ # chart-warning threshold and PR #3840's deliberate choice (PR #3792+ # lowered from 95% → 80%). The total-context branch below uses 95%+ # because it's a stricter condition (input+output combined).if prompt_eval_count >= self.context_limit * 0.80:self.context_truncated = True- # Calculate tokens lost. Prefer original_prompt_estimate when- # it's larger (rare — only when raw prompt text exceeds what- # the model processed). Otherwise use the overflow past- # context_limit as a minimum estimate.if self.original_prompt_estimate > prompt_eval_count:self.tokens_truncated = max(0,@@ -266,10 +280,41 @@ class TokenCountingCallback(BaseCallbackHandler):)logger.warning(- f"Context overflow detected [{source}]! "- f"Prompt tokens: {prompt_eval_count}/{self.context_limit} "- f"(estimated {self.tokens_truncated} tokens truncated, "- f"{self.truncation_ratio:.1%} of prompt)"+ f"Context overflow detected [provider-confirmed] "+ f"research_id={self.research_id} "+ f"model={self.current_model} "+ f"provider={self.current_provider} "+ f"source={source} "+ f"prompt_tokens={prompt_eval_count} "+ f"context_limit={self.context_limit} "+ f"tokens_truncated={self.tokens_truncated} "+ f"truncation_ratio={self.truncation_ratio:.1%}"+ )++ # Total-context overflow: input + output exceeds 95% of context limit+ elif (+ completion_tokens > 0+ and prompt_eval_count + completion_tokens+ >= self.context_limit * 0.95+ ):+ total = prompt_eval_count + completion_tokens+ self.context_truncated = True+ self.tokens_truncated = max(0, total - self.context_limit)+ self.truncation_ratio = (+ self.tokens_truncated / total if total > 0 else 0+ )+ logger.warning(+ f"Context overflow detected [total-context] "+ f"research_id={self.research_id} "+ f"model={self.current_model} "+ f"provider={self.current_provider} "+ f"source={source} "+ f"prompt_tokens={prompt_eval_count} "+ f"completion_tokens={completion_tokens} "+ f"total_tokens={total} "+ f"context_limit={self.context_limit} "+ f"tokens_truncated={self.tokens_truncated} "+ f"truncation_ratio={self.truncation_ratio:.1%}")else:logger.debug(@@ -317,6 +362,9 @@ class TokenCountingCallback(BaseCallbackHandler):# (input_tokens == prompt_eval_count for Ollama)self._check_context_overflow(usage_meta.get("input_tokens", 0),+ completion_tokens=usage_meta.get(+ "output_tokens", 0+ ),source="usage_metadata",)break@@ -344,12 +392,15 @@ class TokenCountingCallback(BaseCallbackHandler):"eval_duration": resp_meta.get("eval_duration"),}- # Check for context overflow+ # Check for context overflow (input only)prompt_eval_count = resp_meta.get("prompt_eval_count", 0)self._check_context_overflow(prompt_eval_count,+ completion_tokens=resp_meta.get(+ "eval_count", 0+ ),source="response_metadata",)@@ -369,6 +420,66 @@ class TokenCountingCallback(BaseCallbackHandler):if token_usage:break+ # Estimation-based overflow detection for providers that don't echo+ # prompt_eval_count (OpenAI, Anthropic, OpenRouter, etc.). The Ollama+ # path above sets context_truncated=True if it detected provider-+ # confirmed truncation; we only fire here if it didn't. For hosted+ # providers the API typically rejects oversize prompts rather than+ # silently truncating, so this signal flags "would-overflow per our+ # estimate" — not the same as actual truncation, hence the [estimated]+ # tag in the log message.+ if not self.context_truncated and self.context_limit:+ # Input-only overflow: prompt estimate exceeds context limit+ if self.original_prompt_estimate > self.context_limit:+ self.context_truncated = True+ self.tokens_truncated = max(+ 0, self.original_prompt_estimate - self.context_limit+ )+ self.truncation_ratio = (+ self.tokens_truncated / self.original_prompt_estimate+ if self.original_prompt_estimate > 0+ else 0+ )+ logger.warning(+ "Context overflow detected [estimated] "+ f"research_id={self.research_id} "+ f"model={self.current_model} "+ f"provider={self.current_provider} "+ f"estimated_prompt_tokens={self.original_prompt_estimate} "+ f"context_limit={self.context_limit} "+ f"tokens_truncated={self.tokens_truncated} "+ f"truncation_ratio={self.truncation_ratio:.1%}"+ )+ # Total-context overflow: input + output exceeds context limit.+ # The "[estimated-total-context]" tag below refers to the+ # *detection method* (post-loop fallback path that runs when+ # _check_context_overflow couldn't fire), not the data source —+ # the prompt_tokens / completion_tokens here come from the+ # provider via response.llm_output.token_usage and are actual+ # counts, not character-based estimates.+ elif token_usage and isinstance(token_usage, dict):+ prompt_tokens = token_usage.get("prompt_tokens", 0)+ completion_tokens = token_usage.get("completion_tokens", 0)+ total = prompt_tokens + completion_tokens+ if total >= self.context_limit * 0.95:+ self.context_truncated = True+ self.tokens_truncated = max(0, total - self.context_limit)+ self.truncation_ratio = (+ self.tokens_truncated / total if total > 0 else 0+ )+ logger.warning(+ "Context overflow detected [estimated-total-context] "+ f"research_id={self.research_id} "+ f"model={self.current_model} "+ f"provider={self.current_provider} "+ f"prompt_tokens={prompt_tokens} "+ f"completion_tokens={completion_tokens} "+ f"total_tokens={total} "+ f"context_limit={self.context_limit} "+ f"tokens_truncated={self.tokens_truncated} "+ f"truncation_ratio={self.truncation_ratio:.1%}"+ )+if token_usage and isinstance(token_usage, dict):prompt_tokens = token_usage.get("prompt_tokens", 0)completion_tokens = token_usage.get("completion_tokens", 0)diff --git a/src/local_deep_research/web/warning_checks/hardware.py b/src/local_deep_research/web/warning_checks/hardware.pyindex 577ea53821..846727b1ba 100644--- a/src/local_deep_research/web/warning_checks/hardware.py+++ b/src/local_deep_research/web/warning_checks/hardware.py@@ -33,6 +33,8 @@ def check_high_context(f"to monitor actual token usage and VRAM consumption."),"dismissKey": "app.warnings.dismiss_high_context",+ "actionUrl": "/metrics/context-overflow",+ "actionLabel": "View context metrics",}diff --git a/tests/metrics/test_token_counter_logic.py b/tests/metrics/test_token_counter_logic.pyindex f52def0e32..1eb5a4e145 100644--- a/tests/metrics/test_token_counter_logic.py+++ b/tests/metrics/test_token_counter_logic.py@@ -371,11 +371,17 @@ class TestContextOverflowDetection:assert cb.tokens_truncated == 1200 - 960def test_no_overflow_below_threshold(self):- """No overflow when below 80% threshold."""+ """No overflow when below 80% threshold AND estimate fits limit.++ Both detection paths must NOT fire: prompt_eval_count is 70% of+ the limit (below the 80% provider-confirmed threshold), and+ original_prompt_estimate is below context_limit (so the+ estimation-based path added in PR #3791 also stays quiet).+ """cb = self._trigger_overflow(context_limit=1000,prompt_eval_count=700, # 70% < 80%- original_prompt_estimate=1200,+ original_prompt_estimate=900, # below limit, no estimation path)assert cb.context_truncated is Falsediff --git a/tests/test_context_overflow_detection.py b/tests/test_context_overflow_detection.pyindex e7cc5c9f29..dd5575d99b 100644--- a/tests/test_context_overflow_detection.py+++ b/tests/test_context_overflow_detection.py@@ -193,11 +193,304 @@ class TestContextOverflowDetection:token_callback.on_llm_end(mock_response)- # Verify warning was logged+ # Verify warning was logged with structured fieldsmock_logger.warning.assert_called()- warning_call = mock_logger.warning.call_args[0][0]+ # The new logger.warning is called with multiple string args that get+ # concatenated by f-strings; inspect each positional arg for the fields.+ warning_call = " ".join(+ str(a) for a in mock_logger.warning.call_args[0]+ )assert "Context overflow detected" in warning_call- assert "2000/2048" in warning_call+ assert "[provider-confirmed]" in warning_call+ assert "prompt_tokens=2000" in warning_call+ assert "context_limit=2048" in warning_call++ @patch("local_deep_research.metrics.token_counter.logger")+ def test_estimated_overflow_for_non_ollama_provider(+ self, mock_logger, token_callback+ ):+ """Detection fires from prompt estimate when provider doesn't echo prompt_eval_count."""+ # Use preset_model/preset_provider so on_llm_start initializes by_model.+ token_callback.preset_model = "gpt-4"+ token_callback.preset_provider = "openai"++ # Large prompt: ~10000 tokens (well over 2048 context_limit)+ large_text = "word " * 10000+ token_callback.on_llm_start({}, [large_text])++ # OpenAI-style response: usage_metadata, no prompt_eval_count+ mock_response = Mock()+ mock_response.llm_output = None+ mock_response.generations = [[Mock()]]+ mock_response.generations[0][0].message = Mock()+ mock_response.generations[0][0].message.usage_metadata = {+ "input_tokens": 100,+ "output_tokens": 50,+ "total_tokens": 150,+ }+ mock_response.generations[0][0].message.response_metadata = {}++ token_callback.on_llm_end(mock_response)++ # Estimation path should mark truncation and log+ assert token_callback.context_truncated is True+ assert token_callback.tokens_truncated > 0+ mock_logger.warning.assert_called()+ warning_call = " ".join(+ str(a) for a in mock_logger.warning.call_args[0]+ )+ assert "[estimated]" in warning_call+ assert "provider=openai" in warning_call+ assert "context_limit=2048" in warning_call++ @patch("local_deep_research.metrics.token_counter.logger")+ def test_provider_confirmed_total_context_overflow(+ self, mock_logger, token_callback+ ):+ """[total-context] fires when input+output exceeds limit but input alone doesn't.++ Input must stay strictly under the input-only threshold (80% of+ context_limit, set by PR #3792 / #3840) so the input-only branch+ does NOT fire and we exercise the total-context elif path.+ """+ large_text = "word " * 500+ token_callback.on_llm_start({}, [large_text])++ # Input is below 80% of 2048 (1638); but input + output >= 95% of 2048 (1945).+ mock_response = Mock()+ mock_response.llm_output = None+ mock_response.generations = [[Mock()]]+ mock_response.generations[0][0].message = Mock()+ mock_response.generations[0][0].message.usage_metadata = None+ mock_response.generations[0][0].message.response_metadata = {+ "prompt_eval_count": 1500, # 73% of 2048 — below 80% input-only threshold+ "eval_count": 500, # 1500+500=2000 >= 95% of 2048 (1945)+ "total_duration": 3000000000,+ }++ token_callback.on_llm_end(mock_response)++ assert token_callback.context_truncated is True+ mock_logger.warning.assert_called()+ warning_call = " ".join(+ str(a) for a in mock_logger.warning.call_args[0]+ )+ assert "[total-context]" in warning_call+ assert "prompt_tokens=1500" in warning_call+ assert "completion_tokens=500" in warning_call+ assert "total_tokens=2000" in warning_call+ assert "context_limit=2048" in warning_call++ @patch("local_deep_research.metrics.token_counter.logger")+ def test_estimated_total_context_overflow_for_non_ollama(+ self, mock_logger, token_callback+ ):+ """[total-context] fires when hosted provider input+output exceeds limit.++ Input alone stays below 80% of 2048 (input-only threshold, set by+ #3792/#3840) so the elif total-context branch is exercised.+ """+ token_callback.preset_model = "gpt-4"+ token_callback.preset_provider = "openai"++ # Prompt estimate below context_limit but not by much+ medium_text = "word " * 1500 # ~1875 estimated tokens+ token_callback.on_llm_start({}, [medium_text])++ # Response reports actual tokens: input below 80% but input+output >= 95%.+ mock_response = Mock()+ mock_response.llm_output = None+ mock_response.generations = [[Mock()]]+ mock_response.generations[0][0].message = Mock()+ mock_response.generations[0][0].message.usage_metadata = {+ "input_tokens": 1500, # 73% of 2048 — below 80% input-only threshold+ "output_tokens": 600, # 1500+600=2100 > 2048 (so tokens_truncated > 0)+ "total_tokens": 2100,+ }+ mock_response.generations[0][0].message.response_metadata = {}++ token_callback.on_llm_end(mock_response)++ assert token_callback.context_truncated is True+ assert token_callback.tokens_truncated > 0+ mock_logger.warning.assert_called()+ warning_call = " ".join(+ str(a) for a in mock_logger.warning.call_args[0]+ )+ assert "[total-context]" in warning_call+ assert "prompt_tokens=1500" in warning_call+ assert "completion_tokens=600" in warning_call+ assert "total_tokens=2100" in warning_call+ assert "context_limit=2048" in warning_call++ @patch("local_deep_research.metrics.token_counter.logger")+ def test_estimated_total_context_overflow_via_llm_output(+ self, mock_logger, token_callback+ ):+ """[estimated-total-context] fires when token_usage comes from llm_output.++ Distinct from the [total-context] path: this exercises the post-loop+ estimation block (token_counter.py ~line 440) where token_usage is+ sourced from response.llm_output rather than from the per-generation+ usage_metadata that _check_context_overflow consumes.++ Conditions for this path:+ - token_usage populated from llm_output (not from generations)+ - generations have NO usage_metadata AND NO response_metadata+ (so _check_context_overflow is never called)+ - original_prompt_estimate <= context_limit (so the [estimated]+ input-only branch above doesn't fire)+ - prompt_tokens + completion_tokens >= 95% of context_limit+ """+ token_callback.preset_model = "gpt-4"+ token_callback.preset_provider = "openai"++ # Estimate well below context_limit — input-only [estimated] stays quiet.+ small_text = "word " * 100 # ~125 estimated tokens+ token_callback.on_llm_start({}, [small_text])++ # token_usage flows in via llm_output; generations carry no metadata+ # so _check_context_overflow is never called.+ mock_response = Mock()+ mock_response.llm_output = {+ "token_usage": {+ "prompt_tokens": 1500, # 73% of 2048 — below 80%+ "completion_tokens": 600, # 1500+600=2100, past context_limit+ "total_tokens": 2100,+ }+ }+ # Empty generations list → loop never enters _check_context_overflow.+ mock_response.generations = []++ token_callback.on_llm_end(mock_response)++ assert token_callback.context_truncated is True+ assert token_callback.tokens_truncated > 0+ mock_logger.warning.assert_called()+ warning_call = " ".join(+ str(a) for a in mock_logger.warning.call_args[0]+ )+ assert "[estimated-total-context]" in warning_call+ assert "prompt_tokens=1500" in warning_call+ assert "completion_tokens=600" in warning_call+ assert "total_tokens=2100" in warning_call+ assert "context_limit=2048" in warning_call++ @patch("local_deep_research.metrics.token_counter.logger")+ def test_estimated_path_fires_on_subsequent_calls_after_first_truncation(+ self, mock_logger, token_callback+ ):+ """Regression: TokenCountingCallback is reused across LLM calls in a+ research session (see config/llm_config.py wrap_llm). on_llm_start+ must reset context_truncated/tokens_truncated/truncation_ratio,+ otherwise the post-loop estimation block's `if not context_truncated`+ guard silently disables [estimated] / [estimated-total-context] for+ every call after the first one that truncates.+ """+ token_callback.preset_model = "gpt-4"+ token_callback.preset_provider = "openai"++ def llm_output_overflow_response():+ mock_response = Mock()+ mock_response.llm_output = {+ "token_usage": {+ "prompt_tokens": 1500,+ "completion_tokens": 600,+ "total_tokens": 2100,+ }+ }+ mock_response.generations = []+ return mock_response++ # Call 1: triggers [estimated-total-context] → context_truncated=True.+ token_callback.on_llm_start({}, ["word " * 100])+ token_callback.on_llm_end(llm_output_overflow_response())+ first_call_count = mock_logger.warning.call_count+ assert first_call_count >= 1+ assert token_callback.context_truncated is True++ # Call 2: same overflow shape — must ALSO log, not be silenced by+ # leftover state from Call 1.+ token_callback.on_llm_start({}, ["word " * 100])+ # Reset asserted by the second on_llm_start clearing prior state:+ assert token_callback.context_truncated is False+ token_callback.on_llm_end(llm_output_overflow_response())+ assert mock_logger.warning.call_count > first_call_count+ assert token_callback.context_truncated is True++ @patch("local_deep_research.metrics.token_counter.logger")+ def test_estimated_overflow_skipped_when_context_limit_none(+ self, mock_logger, token_callback+ ):+ """Estimation path should not fire when context_limit is not set."""+ token_callback.preset_model = "gpt-4"+ token_callback.preset_provider = "openai"+ # Remove context_limit+ token_callback.research_context = {}++ large_text = "word " * 10000+ token_callback.on_llm_start({}, [large_text])++ mock_response = Mock()+ mock_response.llm_output = None+ mock_response.generations = [[Mock()]]+ mock_response.generations[0][0].message = Mock()+ mock_response.generations[0][0].message.usage_metadata = {+ "input_tokens": 100,+ "output_tokens": 50,+ "total_tokens": 150,+ }+ mock_response.generations[0][0].message.response_metadata = {}++ token_callback.on_llm_end(mock_response)++ assert token_callback.context_truncated is False++ @patch("local_deep_research.metrics.token_counter.logger")+ def test_estimated_overflow_zero_prompt_estimate(+ self, mock_logger, token_callback+ ):+ """Estimation path should not fire when prompt estimate is 0."""+ token_callback.preset_model = "gpt-4"+ token_callback.preset_provider = "openai"+ token_callback.original_prompt_estimate = 0++ mock_response = Mock()+ mock_response.llm_output = None+ mock_response.generations = [[Mock()]]+ mock_response.generations[0][0].message = Mock()+ mock_response.generations[0][0].message.usage_metadata = {+ "input_tokens": 100,+ "output_tokens": 50,+ "total_tokens": 150,+ }+ mock_response.generations[0][0].message.response_metadata = {}++ token_callback.on_llm_end(mock_response)++ assert token_callback.context_truncated is False+ assert token_callback.tokens_truncated == 0++ @patch("local_deep_research.metrics.token_counter.logger")+ def test_estimated_overflow_without_on_llm_start(+ self, mock_logger, token_callback+ ):+ """Estimation path should not crash if on_llm_start was never called."""+ # Don't call on_llm_start — fields stay at defaults+ mock_response = Mock()+ mock_response.llm_output = None+ mock_response.generations = [[Mock()]]+ mock_response.generations[0][0].message = Mock()+ mock_response.generations[0][0].message.usage_metadata = {+ "input_tokens": 100,+ "output_tokens": 50,+ "total_tokens": 150,+ }+ mock_response.generations[0][0].message.response_metadata = {}++ # Should not raise and should not mark truncation+ token_callback.on_llm_end(mock_response)+ assert token_callback.context_truncated is Falsedef test_token_usage_save_with_overflow_fields(self, db_session, monkeypatchdiff --git a/tests/web/warning_checks/test_hardware.py b/tests/web/warning_checks/test_hardware.pyindex 931c9d34d9..201fdaac58 100644--- a/tests/web/warning_checks/test_hardware.py+++ b/tests/web/warning_checks/test_hardware.py@@ -64,6 +64,8 @@ class TestCheckHighContext:"title","message","dismissKey",+ "actionUrl",+ "actionLabel",}def test_dismiss_key(self):