Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

feat(metrics): structured truncation log + estimation-based detection (#3791)

LearningCircuit4c1c8d33f23f
diff --git a/changelog.d/+context-overflow-log-fields.feature.md b/changelog.d/+context-overflow-log-fields.feature.md
new file mode 100644
index 0000000000..23ad101548
--- /dev/null
+++ b/changelog.d/+context-overflow-log-fields.feature.md
@@ -0,0 +1 @@
+Context-overflow warnings now include structured fields (`research_id`, `model`, `provider`) and detect both input-only and total-context (input+output) overflow. Four distinct tags (`[provider-confirmed]`, `[total-context]`, `[estimated]`, `[estimated-total-context]`) make different overflow causes distinguishable in logs and alerts. Hosted providers (OpenAI, Anthropic, OpenRouter) get estimation-based detection so their truncation events are no longer invisible. The high-context VRAM warning now links directly to the context-overflow metrics page.
diff --git a/src/local_deep_research/metrics/token_counter.py b/src/local_deep_research/metrics/token_counter.py
index 29d3f41515..e40d18667b 100644
--- a/src/local_deep_research/metrics/token_counter.py
+++ b/src/local_deep_research/metrics/token_counter.py
@@ -74,6 +74,16 @@ class TokenCountingCallback(BaseCallbackHandler):
# Phase 1 Enhancement: Start timing
self.start_time = time.time()
+ # Reset per-call truncation state. The callback instance is shared
+ # across every LLM call in a research session (see llm_config.py
+ # wrap_llm), so without this reset the post-loop estimation block's
+ # `if not self.context_truncated` guard would silently disable
+ # [estimated] / [estimated-total-context] detection on every call
+ # after the first one that truncates.
+ self.context_truncated = False
+ self.tokens_truncated = 0
+ self.truncation_ratio = 0.0
+
# Estimate original prompt size (rough estimate: ~4 chars per token)
if prompts:
total_chars = sum(len(prompt) for prompt in prompts)
@@ -221,31 +231,35 @@ class TokenCountingCallback(BaseCallbackHandler):
self.counts["by_model"][self.current_model]["calls"] += 1
def _check_context_overflow(
- self, prompt_eval_count: int, source: str = ""
+ self,
+ prompt_eval_count: int,
+ completion_tokens: int = 0,
+ source: str = "",
) -> None:
- """Check for context overflow based on prompt token usage.
+ """Check for context overflow based on prompt and total token usage.
Args:
prompt_eval_count: Number of tokens the model actually processed.
+ completion_tokens: Number of tokens generated (for total-context check).
source: Which branch provided the data (for logging).
"""
logger.debug(
f"Context overflow check [{source}]: "
f"prompt_eval_count={prompt_eval_count}, "
+ f"completion_tokens={completion_tokens}, "
f"context_limit={self.context_limit}"
)
if not self.context_limit or prompt_eval_count <= 0:
return
- # Flag when utilization >= 80% or hard overflow
+ # Input-only overflow: prompt at >= 80% of context limit. Matches the
+ # chart-warning threshold and PR #3840's deliberate choice (PR #3792
+ # lowered from 95% → 80%). The total-context branch below uses 95%
+ # because it's a stricter condition (input+output combined).
if prompt_eval_count >= self.context_limit * 0.80:
self.context_truncated = True
- # Calculate tokens lost. Prefer original_prompt_estimate when
- # it's larger (rare — only when raw prompt text exceeds what
- # the model processed). Otherwise use the overflow past
- # context_limit as a minimum estimate.
if self.original_prompt_estimate > prompt_eval_count:
self.tokens_truncated = max(
0,
@@ -266,10 +280,41 @@ class TokenCountingCallback(BaseCallbackHandler):
)
logger.warning(
- f"Context overflow detected [{source}]! "
- f"Prompt tokens: {prompt_eval_count}/{self.context_limit} "
- f"(estimated {self.tokens_truncated} tokens truncated, "
- f"{self.truncation_ratio:.1%} of prompt)"
+ f"Context overflow detected [provider-confirmed] "
+ f"research_id={self.research_id} "
+ f"model={self.current_model} "
+ f"provider={self.current_provider} "
+ f"source={source} "
+ f"prompt_tokens={prompt_eval_count} "
+ f"context_limit={self.context_limit} "
+ f"tokens_truncated={self.tokens_truncated} "
+ f"truncation_ratio={self.truncation_ratio:.1%}"
+ )
+
+ # Total-context overflow: input + output exceeds 95% of context limit
+ elif (
+ completion_tokens > 0
+ and prompt_eval_count + completion_tokens
+ >= self.context_limit * 0.95
+ ):
+ total = prompt_eval_count + completion_tokens
+ self.context_truncated = True
+ self.tokens_truncated = max(0, total - self.context_limit)
+ self.truncation_ratio = (
+ self.tokens_truncated / total if total > 0 else 0
+ )
+ logger.warning(
+ f"Context overflow detected [total-context] "
+ f"research_id={self.research_id} "
+ f"model={self.current_model} "
+ f"provider={self.current_provider} "
+ f"source={source} "
+ f"prompt_tokens={prompt_eval_count} "
+ f"completion_tokens={completion_tokens} "
+ f"total_tokens={total} "
+ f"context_limit={self.context_limit} "
+ f"tokens_truncated={self.tokens_truncated} "
+ f"truncation_ratio={self.truncation_ratio:.1%}"
)
else:
logger.debug(
@@ -317,6 +362,9 @@ class TokenCountingCallback(BaseCallbackHandler):
# (input_tokens == prompt_eval_count for Ollama)
self._check_context_overflow(
usage_meta.get("input_tokens", 0),
+ completion_tokens=usage_meta.get(
+ "output_tokens", 0
+ ),
source="usage_metadata",
)
break
@@ -344,12 +392,15 @@ class TokenCountingCallback(BaseCallbackHandler):
"eval_duration": resp_meta.get("eval_duration"),
}
- # Check for context overflow
+ # Check for context overflow (input only)
prompt_eval_count = resp_meta.get(
"prompt_eval_count", 0
)
self._check_context_overflow(
prompt_eval_count,
+ completion_tokens=resp_meta.get(
+ "eval_count", 0
+ ),
source="response_metadata",
)
@@ -369,6 +420,66 @@ class TokenCountingCallback(BaseCallbackHandler):
if token_usage:
break
+ # Estimation-based overflow detection for providers that don't echo
+ # prompt_eval_count (OpenAI, Anthropic, OpenRouter, etc.). The Ollama
+ # path above sets context_truncated=True if it detected provider-
+ # confirmed truncation; we only fire here if it didn't. For hosted
+ # providers the API typically rejects oversize prompts rather than
+ # silently truncating, so this signal flags "would-overflow per our
+ # estimate" — not the same as actual truncation, hence the [estimated]
+ # tag in the log message.
+ if not self.context_truncated and self.context_limit:
+ # Input-only overflow: prompt estimate exceeds context limit
+ if self.original_prompt_estimate > self.context_limit:
+ self.context_truncated = True
+ self.tokens_truncated = max(
+ 0, self.original_prompt_estimate - self.context_limit
+ )
+ self.truncation_ratio = (
+ self.tokens_truncated / self.original_prompt_estimate
+ if self.original_prompt_estimate > 0
+ else 0
+ )
+ logger.warning(
+ "Context overflow detected [estimated] "
+ f"research_id={self.research_id} "
+ f"model={self.current_model} "
+ f"provider={self.current_provider} "
+ f"estimated_prompt_tokens={self.original_prompt_estimate} "
+ f"context_limit={self.context_limit} "
+ f"tokens_truncated={self.tokens_truncated} "
+ f"truncation_ratio={self.truncation_ratio:.1%}"
+ )
+ # Total-context overflow: input + output exceeds context limit.
+ # The "[estimated-total-context]" tag below refers to the
+ # *detection method* (post-loop fallback path that runs when
+ # _check_context_overflow couldn't fire), not the data source —
+ # the prompt_tokens / completion_tokens here come from the
+ # provider via response.llm_output.token_usage and are actual
+ # counts, not character-based estimates.
+ elif token_usage and isinstance(token_usage, dict):
+ prompt_tokens = token_usage.get("prompt_tokens", 0)
+ completion_tokens = token_usage.get("completion_tokens", 0)
+ total = prompt_tokens + completion_tokens
+ if total >= self.context_limit * 0.95:
+ self.context_truncated = True
+ self.tokens_truncated = max(0, total - self.context_limit)
+ self.truncation_ratio = (
+ self.tokens_truncated / total if total > 0 else 0
+ )
+ logger.warning(
+ "Context overflow detected [estimated-total-context] "
+ f"research_id={self.research_id} "
+ f"model={self.current_model} "
+ f"provider={self.current_provider} "
+ f"prompt_tokens={prompt_tokens} "
+ f"completion_tokens={completion_tokens} "
+ f"total_tokens={total} "
+ f"context_limit={self.context_limit} "
+ f"tokens_truncated={self.tokens_truncated} "
+ f"truncation_ratio={self.truncation_ratio:.1%}"
+ )
+
if token_usage and isinstance(token_usage, dict):
prompt_tokens = token_usage.get("prompt_tokens", 0)
completion_tokens = token_usage.get("completion_tokens", 0)
diff --git a/src/local_deep_research/web/warning_checks/hardware.py b/src/local_deep_research/web/warning_checks/hardware.py
index 577ea53821..846727b1ba 100644
--- a/src/local_deep_research/web/warning_checks/hardware.py
+++ b/src/local_deep_research/web/warning_checks/hardware.py
@@ -33,6 +33,8 @@ def check_high_context(
f"to monitor actual token usage and VRAM consumption."
),
"dismissKey": "app.warnings.dismiss_high_context",
+ "actionUrl": "/metrics/context-overflow",
+ "actionLabel": "View context metrics",
}
diff --git a/tests/metrics/test_token_counter_logic.py b/tests/metrics/test_token_counter_logic.py
index f52def0e32..1eb5a4e145 100644
--- a/tests/metrics/test_token_counter_logic.py
+++ b/tests/metrics/test_token_counter_logic.py
@@ -371,11 +371,17 @@ class TestContextOverflowDetection:
assert cb.tokens_truncated == 1200 - 960
def test_no_overflow_below_threshold(self):
- """No overflow when below 80% threshold."""
+ """No overflow when below 80% threshold AND estimate fits limit.
+
+ Both detection paths must NOT fire: prompt_eval_count is 70% of
+ the limit (below the 80% provider-confirmed threshold), and
+ original_prompt_estimate is below context_limit (so the
+ estimation-based path added in PR #3791 also stays quiet).
+ """
cb = self._trigger_overflow(
context_limit=1000,
prompt_eval_count=700, # 70% < 80%
- original_prompt_estimate=1200,
+ original_prompt_estimate=900, # below limit, no estimation path
)
assert cb.context_truncated is False
diff --git a/tests/test_context_overflow_detection.py b/tests/test_context_overflow_detection.py
index e7cc5c9f29..dd5575d99b 100644
--- a/tests/test_context_overflow_detection.py
+++ b/tests/test_context_overflow_detection.py
@@ -193,11 +193,304 @@ class TestContextOverflowDetection:
token_callback.on_llm_end(mock_response)
- # Verify warning was logged
+ # Verify warning was logged with structured fields
mock_logger.warning.assert_called()
- warning_call = mock_logger.warning.call_args[0][0]
+ # The new logger.warning is called with multiple string args that get
+ # concatenated by f-strings; inspect each positional arg for the fields.
+ warning_call = " ".join(
+ str(a) for a in mock_logger.warning.call_args[0]
+ )
assert "Context overflow detected" in warning_call
- assert "2000/2048" in warning_call
+ assert "[provider-confirmed]" in warning_call
+ assert "prompt_tokens=2000" in warning_call
+ assert "context_limit=2048" in warning_call
+
+ @patch("local_deep_research.metrics.token_counter.logger")
+ def test_estimated_overflow_for_non_ollama_provider(
+ self, mock_logger, token_callback
+ ):
+ """Detection fires from prompt estimate when provider doesn't echo prompt_eval_count."""
+ # Use preset_model/preset_provider so on_llm_start initializes by_model.
+ token_callback.preset_model = "gpt-4"
+ token_callback.preset_provider = "openai"
+
+ # Large prompt: ~10000 tokens (well over 2048 context_limit)
+ large_text = "word " * 10000
+ token_callback.on_llm_start({}, [large_text])
+
+ # OpenAI-style response: usage_metadata, no prompt_eval_count
+ mock_response = Mock()
+ mock_response.llm_output = None
+ mock_response.generations = [[Mock()]]
+ mock_response.generations[0][0].message = Mock()
+ mock_response.generations[0][0].message.usage_metadata = {
+ "input_tokens": 100,
+ "output_tokens": 50,
+ "total_tokens": 150,
+ }
+ mock_response.generations[0][0].message.response_metadata = {}
+
+ token_callback.on_llm_end(mock_response)
+
+ # Estimation path should mark truncation and log
+ assert token_callback.context_truncated is True
+ assert token_callback.tokens_truncated > 0
+ mock_logger.warning.assert_called()
+ warning_call = " ".join(
+ str(a) for a in mock_logger.warning.call_args[0]
+ )
+ assert "[estimated]" in warning_call
+ assert "provider=openai" in warning_call
+ assert "context_limit=2048" in warning_call
+
+ @patch("local_deep_research.metrics.token_counter.logger")
+ def test_provider_confirmed_total_context_overflow(
+ self, mock_logger, token_callback
+ ):
+ """[total-context] fires when input+output exceeds limit but input alone doesn't.
+
+ Input must stay strictly under the input-only threshold (80% of
+ context_limit, set by PR #3792 / #3840) so the input-only branch
+ does NOT fire and we exercise the total-context elif path.
+ """
+ large_text = "word " * 500
+ token_callback.on_llm_start({}, [large_text])
+
+ # Input is below 80% of 2048 (1638); but input + output >= 95% of 2048 (1945).
+ mock_response = Mock()
+ mock_response.llm_output = None
+ mock_response.generations = [[Mock()]]
+ mock_response.generations[0][0].message = Mock()
+ mock_response.generations[0][0].message.usage_metadata = None
+ mock_response.generations[0][0].message.response_metadata = {
+ "prompt_eval_count": 1500, # 73% of 2048 — below 80% input-only threshold
+ "eval_count": 500, # 1500+500=2000 >= 95% of 2048 (1945)
+ "total_duration": 3000000000,
+ }
+
+ token_callback.on_llm_end(mock_response)
+
+ assert token_callback.context_truncated is True
+ mock_logger.warning.assert_called()
+ warning_call = " ".join(
+ str(a) for a in mock_logger.warning.call_args[0]
+ )
+ assert "[total-context]" in warning_call
+ assert "prompt_tokens=1500" in warning_call
+ assert "completion_tokens=500" in warning_call
+ assert "total_tokens=2000" in warning_call
+ assert "context_limit=2048" in warning_call
+
+ @patch("local_deep_research.metrics.token_counter.logger")
+ def test_estimated_total_context_overflow_for_non_ollama(
+ self, mock_logger, token_callback
+ ):
+ """[total-context] fires when hosted provider input+output exceeds limit.
+
+ Input alone stays below 80% of 2048 (input-only threshold, set by
+ #3792/#3840) so the elif total-context branch is exercised.
+ """
+ token_callback.preset_model = "gpt-4"
+ token_callback.preset_provider = "openai"
+
+ # Prompt estimate below context_limit but not by much
+ medium_text = "word " * 1500 # ~1875 estimated tokens
+ token_callback.on_llm_start({}, [medium_text])
+
+ # Response reports actual tokens: input below 80% but input+output >= 95%.
+ mock_response = Mock()
+ mock_response.llm_output = None
+ mock_response.generations = [[Mock()]]
+ mock_response.generations[0][0].message = Mock()
+ mock_response.generations[0][0].message.usage_metadata = {
+ "input_tokens": 1500, # 73% of 2048 — below 80% input-only threshold
+ "output_tokens": 600, # 1500+600=2100 > 2048 (so tokens_truncated > 0)
+ "total_tokens": 2100,
+ }
+ mock_response.generations[0][0].message.response_metadata = {}
+
+ token_callback.on_llm_end(mock_response)
+
+ assert token_callback.context_truncated is True
+ assert token_callback.tokens_truncated > 0
+ mock_logger.warning.assert_called()
+ warning_call = " ".join(
+ str(a) for a in mock_logger.warning.call_args[0]
+ )
+ assert "[total-context]" in warning_call
+ assert "prompt_tokens=1500" in warning_call
+ assert "completion_tokens=600" in warning_call
+ assert "total_tokens=2100" in warning_call
+ assert "context_limit=2048" in warning_call
+
+ @patch("local_deep_research.metrics.token_counter.logger")
+ def test_estimated_total_context_overflow_via_llm_output(
+ self, mock_logger, token_callback
+ ):
+ """[estimated-total-context] fires when token_usage comes from llm_output.
+
+ Distinct from the [total-context] path: this exercises the post-loop
+ estimation block (token_counter.py ~line 440) where token_usage is
+ sourced from response.llm_output rather than from the per-generation
+ usage_metadata that _check_context_overflow consumes.
+
+ Conditions for this path:
+ - token_usage populated from llm_output (not from generations)
+ - generations have NO usage_metadata AND NO response_metadata
+ (so _check_context_overflow is never called)
+ - original_prompt_estimate <= context_limit (so the [estimated]
+ input-only branch above doesn't fire)
+ - prompt_tokens + completion_tokens >= 95% of context_limit
+ """
+ token_callback.preset_model = "gpt-4"
+ token_callback.preset_provider = "openai"
+
+ # Estimate well below context_limit — input-only [estimated] stays quiet.
+ small_text = "word " * 100 # ~125 estimated tokens
+ token_callback.on_llm_start({}, [small_text])
+
+ # token_usage flows in via llm_output; generations carry no metadata
+ # so _check_context_overflow is never called.
+ mock_response = Mock()
+ mock_response.llm_output = {
+ "token_usage": {
+ "prompt_tokens": 1500, # 73% of 2048 — below 80%
+ "completion_tokens": 600, # 1500+600=2100, past context_limit
+ "total_tokens": 2100,
+ }
+ }
+ # Empty generations list → loop never enters _check_context_overflow.
+ mock_response.generations = []
+
+ token_callback.on_llm_end(mock_response)
+
+ assert token_callback.context_truncated is True
+ assert token_callback.tokens_truncated > 0
+ mock_logger.warning.assert_called()
+ warning_call = " ".join(
+ str(a) for a in mock_logger.warning.call_args[0]
+ )
+ assert "[estimated-total-context]" in warning_call
+ assert "prompt_tokens=1500" in warning_call
+ assert "completion_tokens=600" in warning_call
+ assert "total_tokens=2100" in warning_call
+ assert "context_limit=2048" in warning_call
+
+ @patch("local_deep_research.metrics.token_counter.logger")
+ def test_estimated_path_fires_on_subsequent_calls_after_first_truncation(
+ self, mock_logger, token_callback
+ ):
+ """Regression: TokenCountingCallback is reused across LLM calls in a
+ research session (see config/llm_config.py wrap_llm). on_llm_start
+ must reset context_truncated/tokens_truncated/truncation_ratio,
+ otherwise the post-loop estimation block's `if not context_truncated`
+ guard silently disables [estimated] / [estimated-total-context] for
+ every call after the first one that truncates.
+ """
+ token_callback.preset_model = "gpt-4"
+ token_callback.preset_provider = "openai"
+
+ def llm_output_overflow_response():
+ mock_response = Mock()
+ mock_response.llm_output = {
+ "token_usage": {
+ "prompt_tokens": 1500,
+ "completion_tokens": 600,
+ "total_tokens": 2100,
+ }
+ }
+ mock_response.generations = []
+ return mock_response
+
+ # Call 1: triggers [estimated-total-context] → context_truncated=True.
+ token_callback.on_llm_start({}, ["word " * 100])
+ token_callback.on_llm_end(llm_output_overflow_response())
+ first_call_count = mock_logger.warning.call_count
+ assert first_call_count >= 1
+ assert token_callback.context_truncated is True
+
+ # Call 2: same overflow shape — must ALSO log, not be silenced by
+ # leftover state from Call 1.
+ token_callback.on_llm_start({}, ["word " * 100])
+ # Reset asserted by the second on_llm_start clearing prior state:
+ assert token_callback.context_truncated is False
+ token_callback.on_llm_end(llm_output_overflow_response())
+ assert mock_logger.warning.call_count > first_call_count
+ assert token_callback.context_truncated is True
+
+ @patch("local_deep_research.metrics.token_counter.logger")
+ def test_estimated_overflow_skipped_when_context_limit_none(
+ self, mock_logger, token_callback
+ ):
+ """Estimation path should not fire when context_limit is not set."""
+ token_callback.preset_model = "gpt-4"
+ token_callback.preset_provider = "openai"
+ # Remove context_limit
+ token_callback.research_context = {}
+
+ large_text = "word " * 10000
+ token_callback.on_llm_start({}, [large_text])
+
+ mock_response = Mock()
+ mock_response.llm_output = None
+ mock_response.generations = [[Mock()]]
+ mock_response.generations[0][0].message = Mock()
+ mock_response.generations[0][0].message.usage_metadata = {
+ "input_tokens": 100,
+ "output_tokens": 50,
+ "total_tokens": 150,
+ }
+ mock_response.generations[0][0].message.response_metadata = {}
+
+ token_callback.on_llm_end(mock_response)
+
+ assert token_callback.context_truncated is False
+
+ @patch("local_deep_research.metrics.token_counter.logger")
+ def test_estimated_overflow_zero_prompt_estimate(
+ self, mock_logger, token_callback
+ ):
+ """Estimation path should not fire when prompt estimate is 0."""
+ token_callback.preset_model = "gpt-4"
+ token_callback.preset_provider = "openai"
+ token_callback.original_prompt_estimate = 0
+
+ mock_response = Mock()
+ mock_response.llm_output = None
+ mock_response.generations = [[Mock()]]
+ mock_response.generations[0][0].message = Mock()
+ mock_response.generations[0][0].message.usage_metadata = {
+ "input_tokens": 100,
+ "output_tokens": 50,
+ "total_tokens": 150,
+ }
+ mock_response.generations[0][0].message.response_metadata = {}
+
+ token_callback.on_llm_end(mock_response)
+
+ assert token_callback.context_truncated is False
+ assert token_callback.tokens_truncated == 0
+
+ @patch("local_deep_research.metrics.token_counter.logger")
+ def test_estimated_overflow_without_on_llm_start(
+ self, mock_logger, token_callback
+ ):
+ """Estimation path should not crash if on_llm_start was never called."""
+ # Don't call on_llm_start — fields stay at defaults
+ mock_response = Mock()
+ mock_response.llm_output = None
+ mock_response.generations = [[Mock()]]
+ mock_response.generations[0][0].message = Mock()
+ mock_response.generations[0][0].message.usage_metadata = {
+ "input_tokens": 100,
+ "output_tokens": 50,
+ "total_tokens": 150,
+ }
+ mock_response.generations[0][0].message.response_metadata = {}
+
+ # Should not raise and should not mark truncation
+ token_callback.on_llm_end(mock_response)
+ assert token_callback.context_truncated is False
def test_token_usage_save_with_overflow_fields(
self, db_session, monkeypatch
diff --git a/tests/web/warning_checks/test_hardware.py b/tests/web/warning_checks/test_hardware.py
index 931c9d34d9..201fdaac58 100644
--- a/tests/web/warning_checks/test_hardware.py
+++ b/tests/web/warning_checks/test_hardware.py
@@ -64,6 +64,8 @@ class TestCheckHighContext:
"title",
"message",
"dismissKey",
+ "actionUrl",
+ "actionLabel",
}
def test_dismiss_key(self):