Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(metrics): populate call_stack + JSON-decode it in the timeline endpoint (#5173)

Sdbd9aa8f1ddb
diff --git a/changelog.d/5173.bugfix.md b/changelog.d/5173.bugfix.md
new file mode 100644
index 0000000000..77fc75b44c
--- /dev/null
+++ b/changelog.d/5173.bugfix.md
@@ -0,0 +1,8 @@
+**Fix "LLM Call Stack Analysis" panel showing literal `null`**.
+The metrics timeline endpoint now decodes the `call_stack` JSON column
+into the actual `string | null` value the UI expects, so the
+"LLM Call Stack Analysis" panel on `/details/<research_id>` falls back
+to "No stack trace available" instead of rendering the literal text
+`null`. Call-stack capture is also keyed by `run_id` (and cleared on
+end/error), so overlapping or unmatched LLM calls no longer inherit a
+previous run's stack.
diff --git a/src/local_deep_research/metrics/token_counter.py b/src/local_deep_research/metrics/token_counter.py
index 763a438425..1fa03c9509 100644
--- a/src/local_deep_research/metrics/token_counter.py
+++ b/src/local_deep_research/metrics/token_counter.py
@@ -2,7 +2,9 @@
import inspect
import json
+import threading
import time
+import uuid
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Optional
@@ -19,6 +21,12 @@ from .query_utils import (
get_time_filter_condition,
)
+# Maximum number of frames captured per LLM call when building the
+# simplified call stack shown in the metrics UI. Five frames is enough to
+# surface the LDR call site and the immediate LangChain/LangGraph frames
+# above it without flooding the persisted column.
+_CALL_STACK_FRAME_LIMIT = 5
+
class TokenCountingCallback(BaseCallbackHandler):
"""Callback handler for counting tokens across different models."""
@@ -48,7 +56,20 @@ class TokenCountingCallback(BaseCallbackHandler):
self.success_status = "success"
self.error_type = None
- # Call stack tracking
+ # Per-run call metadata. The callback is shared across every LLM
+ # call in a research session (see llm_config.py wrap_llm), and a
+ # single session can drive overlapping chains (e.g. agent + judge
+ # sub-strategies). Storing the captured calling_file /
+ # calling_function / call_stack on instance attributes caused
+ # overlapping or unmatched runs to inherit each other's stack
+ # (or to keep a previous run's stack when the new run had no
+ # matching LDR frame). Keying by langchain's run_id and removing
+ # the entry on end/error bounds the metadata to its call.
+ self._call_meta: Dict[str, Dict[str, Any]] = {}
+ self._lock = threading.Lock()
+ # Mirror the most recently captured values for code paths (and
+ # legacy tests) that read these as plain attributes. They are not
+ # authoritative; the per-run dict is.
self.calling_file = None
self.calling_function = None
self.call_stack = None
@@ -107,20 +128,32 @@ class TokenCountingCallback(BaseCallbackHandler):
# Get context limit from research context (will be set from settings)
self.context_limit = self.research_context.get("context_limit")
+ # Per-run call metadata, keyed by langchain's run_id so overlapping
+ # runs (or a no-match run following a match) don't see each other's
+ # stack. The run_id comes in via kwargs from langchain; fall back
+ # to a synthetic key when invoked outside the normal chain so
+ # tests / direct callers still work.
+ run_id = kwargs.get("run_id") or f"_synthetic-{uuid.uuid4()}"
+
# Phase 1 Enhancement: Capture call stack information
+ calling_file = None
+ calling_function = None
+ call_stack = None
try:
stack = inspect.stack()
# Skip the first few frames (this method, langchain internals)
- # Look for the first frame that's in our project directory
- for frame_info in stack[1:]:
+ # Look for the first frame that's in our project directory.
+ #
+ # Note: the project is installed as a wheel inside
+ # .venv/.../site-packages in the Docker image, so the project
+ # frames DO contain "site-packages" and "venv" as substrings.
+ # The "local_deep_research" package-name check is sufficient to
+ # identify project files — third-party packages with the same
+ # name do not exist.
+ for matched_index, frame_info in enumerate(stack[1:], start=1):
file_path = frame_info.filename
- # Look for any frame containing local_deep_research project
- if (
- "local_deep_research" in file_path
- and "site-packages" not in file_path
- and "venv" not in file_path
- ):
+ if "local_deep_research" in file_path:
# Extract relative path from local_deep_research
if "src/local_deep_research" in file_path:
relative_path = file_path.split(
@@ -138,21 +171,25 @@ class TokenCountingCallback(BaseCallbackHandler):
else:
relative_path = Path(file_path).name
- self.calling_file = relative_path
- self.calling_function = frame_info.function
+ calling_file = relative_path
+ calling_function = frame_info.function
- # Capture a simplified call stack (just the relevant frames)
+ # Capture a simplified call stack starting from the
+ # matched frame onward (limit to _CALL_STACK_FRAME_LIMIT
+ # frames). Previously this sliced stack[1:6]
+ # unconditionally, so when the matched LDR frame sat
+ # deeper than index 5 (common with langgraph),
+ # call_stack_frames only inspected langchain/langgraph
+ # frames and stayed empty.
call_stack_frames = []
- for frame in stack[1:6]: # Limit to 5 frames
- if (
- "local_deep_research" in frame.filename
- and "site-packages" not in frame.filename
- and "venv" not in frame.filename
- ):
+ for frame in stack[
+ matched_index : matched_index + _CALL_STACK_FRAME_LIMIT
+ ]:
+ if "local_deep_research" in frame.filename:
frame_name = f"{Path(frame.filename).name}:{frame.function}:{frame.lineno}"
call_stack_frames.append(frame_name)
- self.call_stack = (
+ call_stack = (
" -> ".join(call_stack_frames)
if call_stack_frames
else None
@@ -162,6 +199,19 @@ class TokenCountingCallback(BaseCallbackHandler):
logger.warning("Error capturing call stack")
# Continue without call stack info if there's an error
+ with self._lock:
+ self._call_meta[run_id] = {
+ "calling_file": calling_file,
+ "calling_function": calling_function,
+ "call_stack": call_stack,
+ }
+ # Mirror to the legacy attributes for callers that read them
+ # directly. The per-run dict is authoritative; these are a
+ # convenience that reflects the most recent on_llm_start.
+ self.calling_file = calling_file
+ self.calling_function = calling_function
+ self.call_stack = call_stack
+
# First, use preset values if available
if self.preset_model:
self.current_model = self.preset_model
@@ -341,6 +391,16 @@ class TokenCountingCallback(BaseCallbackHandler):
if self.start_time:
self.response_time_ms = int((time.time() - self.start_time) * 1000)
+ # Pop the per-run call metadata captured at on_llm_start so a
+ # later unmatched run cannot persist a previous run's stack.
+ # Missing-key case (e.g. direct test invocation that skipped
+ # on_llm_start) yields a None-filled dict, which the API/UI
+ # already treat as "no stack".
+ run_id = kwargs.get("run_id")
+ call_meta = (
+ self._pop_call_meta(run_id) if run_id else self._legacy_meta()
+ )
+
# Extract token usage from response
token_usage = None
@@ -517,7 +577,7 @@ class TokenCountingCallback(BaseCallbackHandler):
# Save to database if we have a research_id
if self.research_id:
- self._save_to_db(prompt_tokens, completion_tokens)
+ self._save_to_db(prompt_tokens, completion_tokens, call_meta)
elif self.research_id:
# The provider returned no usage data at all (e.g. OpenAI-
# compatible servers that omit `usage` on streamed responses,
@@ -545,7 +605,7 @@ class TokenCountingCallback(BaseCallbackHandler):
f"research_id={self.research_id}.{hint} "
f"(further occurrences this session are suppressed)"
)
- self._save_to_db(0, 0)
+ self._save_to_db(0, 0, call_meta)
def on_llm_error(self, error, **kwargs: Any) -> None:
"""Called when LLM encounters an error."""
@@ -556,9 +616,62 @@ class TokenCountingCallback(BaseCallbackHandler):
self.success_status = "error"
self.error_type = str(type(error).__name__)
+ # Pop the per-run call metadata captured at on_llm_start (if any)
+ # so a later unmatched run does not inherit a failed run's stack.
+ run_id = kwargs.get("run_id")
+ call_meta = (
+ self._pop_call_meta(run_id) if run_id else self._legacy_meta()
+ )
+
# Still save to database to track failed calls
if self.research_id:
- self._save_to_db(0, 0)
+ self._save_to_db(0, 0, call_meta)
+
+ def _pop_call_meta(self, run_id: str) -> Dict[str, Any]:
+ """Remove and return the per-run call metadata for ``run_id``.
+
+ Falls back to ``_legacy_meta()`` (the most recently captured
+ values) if the run was never registered — e.g. ``on_llm_error``
+ firing without a corresponding ``on_llm_start`` (provider raised
+ during argument validation) or a test that drives the callbacks
+ out of order. After popping, refresh the legacy mirror
+ attributes from the next-newest remaining entry so direct
+ ``cb.calling_file`` reads in overlapping-run scenarios do not
+ show a deleted run's stack.
+ """
+ with self._lock:
+ meta = self._call_meta.pop(
+ run_id,
+ {
+ "calling_file": None,
+ "calling_function": None,
+ "call_stack": None,
+ },
+ )
+ if self._call_meta:
+ latest_id = next(reversed(self._call_meta))
+ latest = self._call_meta[latest_id]
+ else:
+ latest = meta
+ self.calling_file = latest.get("calling_file")
+ self.calling_function = latest.get("calling_function")
+ self.call_stack = latest.get("call_stack")
+ return meta
+
+ def _legacy_meta(self) -> Dict[str, Any]:
+ """Return the legacy mirror attributes as a meta dict.
+
+ Used by callers that did not register a run (no run_id kwarg),
+ e.g. direct test invocations of ``on_llm_end`` /
+ ``on_llm_error``. The values reflect the most recent
+ ``on_llm_start`` (or whatever the test set explicitly).
+ """
+ with self._lock:
+ return {
+ "calling_file": self.calling_file,
+ "calling_function": self.calling_function,
+ "call_stack": self.call_stack,
+ }
def _get_context_overflow_fields(self) -> Dict[str, Any]:
"""Get context overflow detection fields for database saving."""
@@ -584,8 +697,31 @@ class TokenCountingCallback(BaseCallbackHandler):
"ollama_eval_duration": self.ollama_metrics.get("eval_duration"),
}
- def _save_to_db(self, prompt_tokens: int, completion_tokens: int):
- """Save token usage to the database."""
+ def _save_to_db(
+ self,
+ prompt_tokens: int,
+ completion_tokens: int,
+ call_meta: Optional[Dict[str, Any]] = None,
+ ):
+ """Save token usage to the database.
+
+ Args:
+ prompt_tokens: Prompt-side token count for this call.
+ completion_tokens: Completion-side token count for this call.
+ call_meta: Per-run call metadata captured by on_llm_start
+ (calling_file / calling_function / call_stack). When the
+ callback is invoked outside on_llm_end / on_llm_error
+ (e.g. legacy tests that set the attributes directly),
+ ``call_meta`` may be ``None`` and the instance attributes
+ are used as a fallback.
+ """
+ if call_meta is None:
+ with self._lock:
+ call_meta = {
+ "calling_file": self.calling_file,
+ "calling_function": self.calling_function,
+ "call_stack": self.call_stack,
+ }
# Check if we're in a thread - if so, queue the save for later
import threading
@@ -628,9 +764,9 @@ class TokenCountingCallback(BaseCallbackHandler):
"search_engine_selected": self.research_context.get(
"search_engine_selected"
),
- "calling_file": self.calling_file,
- "calling_function": self.calling_function,
- "call_stack": self.call_stack,
+ "calling_file": call_meta.get("calling_file"),
+ "calling_function": call_meta.get("calling_function"),
+ "call_stack": call_meta.get("call_stack"),
# Add context overflow fields using helper method
**self._get_context_overflow_fields(),
}
@@ -734,9 +870,9 @@ class TokenCountingCallback(BaseCallbackHandler):
search_engines_planned=search_engines_planned,
search_engine_selected=search_engine_selected,
# Phase 1 Enhancement: Call stack tracking
- calling_file=self.calling_file,
- calling_function=self.calling_function,
- call_stack=self.call_stack,
+ calling_file=call_meta.get("calling_file"),
+ calling_function=call_meta.get("calling_function"),
+ call_stack=call_meta.get("call_stack"),
# Add context overflow fields using helper method
**self._get_context_overflow_fields(),
)
@@ -1795,6 +1931,36 @@ class TokenCounter:
cumulative_prompt_tokens += row[2] or 0
cumulative_completion_tokens += row[3] or 0
+ # call_stack is a JSON column. Depending on the
+ # SQLAlchemy/SQLite/driver combo the raw text() SELECT may
+ # either hand back the raw JSON-encoded TEXT or already
+ # the decoded Python value. Normalise to the API/UI
+ # contract (`string | null`): pass through None and
+ # strings; re-serialise any other stored JSON shape
+ # (number / bool / list / object) so a Python int/bool/
+ # list never leaks to the browser. Legacy unparseable
+ # strings pass through as-is.
+ raw_call_stack = row[13]
+ if raw_call_stack is None or raw_call_stack == "":
+ decoded_call_stack = None
+ elif isinstance(raw_call_stack, str):
+ try:
+ decoded = json.loads(raw_call_stack)
+ except (TypeError, ValueError):
+ decoded_call_stack = raw_call_stack
+ else:
+ decoded_call_stack = (
+ decoded
+ if decoded is None or isinstance(decoded, str)
+ else json.dumps(decoded)
+ )
+ else:
+ decoded_call_stack = (
+ raw_call_stack
+ if isinstance(raw_call_stack, str)
+ else json.dumps(raw_call_stack)
+ )
+
timeline.append(
{
"timestamp": str(row[0]) if row[0] else None,
@@ -1813,7 +1979,7 @@ class TokenCounter:
"model_name": row[10],
"calling_file": row[11],
"calling_function": row[12],
- "call_stack": row[13],
+ "call_stack": decoded_call_stack,
}
)
diff --git a/tests/metrics/test_token_counter_callback.py b/tests/metrics/test_token_counter_callback.py
index 91ad82ab34..6fe6d60d38 100644
--- a/tests/metrics/test_token_counter_callback.py
+++ b/tests/metrics/test_token_counter_callback.py
@@ -422,7 +422,8 @@ class TestTokenCountingCallbackMissingUsageData:
callback.on_llm_end(self._response_without_usage())
- callback._save_to_db.assert_called_once_with(0, 0)
+ callback._save_to_db.assert_called_once()
+ assert callback._save_to_db.call_args[0][:2] == (0, 0)
def test_no_usage_without_research_id_does_not_save(self):
"""Without a research_id there is nothing to persist."""
@@ -485,7 +486,8 @@ class TestTokenCountingCallbackMissingUsageData:
response.generations = []
callback.on_llm_end(response)
- callback._save_to_db.assert_called_once_with(10, 5)
+ callback._save_to_db.assert_called_once()
+ assert callback._save_to_db.call_args[0][:2] == (10, 5)
class TestTokenCountingCallbackErrorHandling:
diff --git a/tests/metrics/test_token_counter_coverage.py b/tests/metrics/test_token_counter_coverage.py
index 85caba98f2..e872902fb4 100644
--- a/tests/metrics/test_token_counter_coverage.py
+++ b/tests/metrics/test_token_counter_coverage.py
@@ -4,7 +4,10 @@ import json
import threading
from unittest.mock import MagicMock, Mock, patch
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+from local_deep_research.database.models import Base
from local_deep_research.metrics.token_counter import (
TokenCounter,
TokenCountingCallback,
@@ -578,3 +581,115 @@ class TestGetResearchTimelineMetricsCoverage:
assert result["timeline"] == []
assert result["summary"]["total_calls"] == 0
assert result["phase_stats"] == {}
+
+
+class TestGetResearchTimelineMetricsCallStackDeserialization:
+ """Integration test verifying the timeline endpoint JSON-decodes the
+ call_stack column.
+
+ TokenUsage.call_stack is a SQLite JSON column. The raw text() SELECT
+ used by get_research_timeline_metrics bypasses SQLAlchemy's type
+ deserialization, so the value comes back as the JSON-encoded string
+ (4-char 'null' or quoted string). The endpoint must surface Python
+ None / the unquoted string so the frontend falls back to its
+ 'No stack trace available' message instead of rendering the literal
+ 'null'.
+ """
+
+ def _seed(self, session, research_id, call_stack):
+ from local_deep_research.database.models import (
+ ResearchHistory,
+ TokenUsage,
+ )
+
+ session.add(
+ ResearchHistory(
+ id=research_id,
+ query="q",
+ mode="quick",
+ status="completed",
+ created_at="2026-01-01T00:00:00",
+ )
+ )
+ session.add(
+ TokenUsage(
+ research_id=research_id,
+ model_name="gpt-4",
+ model_provider="openai",
+ prompt_tokens=10,
+ completion_tokens=5,
+ total_tokens=15,
+ calling_file="runner.py",
+ calling_function="execute",
+ call_stack=call_stack,
+ )
+ )
+ session.commit()
+
+ def _run(self, research_id, session_factory):
+ counter = TokenCounter()
+ real_session = session_factory()
+
+ class _Ctx:
+ def __enter__(self_inner):
+ return real_session
+
+ def __exit__(self_inner, *exc):
+ real_session.close()
+ return False
+
+ def _factory(username=None):
+ return _Ctx()
+
+ with patch("flask.session", {"username": "alice"}):
+ with patch(
+ "local_deep_research.database.session_context.get_user_db_session",
+ side_effect=_factory,
+ ):
+ return counter.get_research_timeline_metrics(research_id)
+
+ def test_json_null_call_stack_returned_as_none(self):
+ engine = create_engine("sqlite:///:memory:")
+ Base.metadata.create_all(engine)
+ Session = sessionmaker(bind=engine)
+ with Session() as session:
+ self._seed(session, "res-json-null", None)
+
+ result = self._run("res-json-null", Session)
+ assert len(result["timeline"]) == 1
+ assert result["timeline"][0]["call_stack"] is None
+
+ def test_real_call_stack_string_unquoted(self):
+ engine = create_engine("sqlite:///:memory:")
+ Base.metadata.create_all(engine)
+ Session = sessionmaker(bind=engine)
+ with Session() as session:
+ self._seed(session, "res-real-stack", "a.py:f:1 -> b.py:g:2")
+
+ result = self._run("res-real-stack", Session)
+ assert result["timeline"][0]["call_stack"] == "a.py:f:1 -> b.py:g:2"
+
+ def test_non_string_json_shapes_serialized_back_to_string(self):
+ """Legacy or out-of-band writes can land numbers, bools, lists
+ or objects in the JSON column. The API contract is `string |
+ null`, so the endpoint must serialize those shapes back to a
+ string instead of letting a Python int/bool/list leak to the
+ browser."""
+ for stored, expected in [
+ (123, "123"),
+ (True, "true"),
+ ([1, 2], "[1, 2]"),
+ ({"a": 1}, '{"a": 1}'),
+ ]:
+ engine = create_engine("sqlite:///:memory:")
+ Base.metadata.create_all(engine)
+ Session = sessionmaker(bind=engine)
+ rid = f"res-{type(stored).__name__}"
+ with Session() as session:
+ self._seed(session, rid, stored)
+
+ result = self._run(rid, Session)
+ assert result["timeline"][0]["call_stack"] == expected, (
+ f"stored={stored!r} expected={expected!r} "
+ f"got={result['timeline'][0]['call_stack']!r}"
+ )
diff --git a/tests/metrics/test_token_counter_deep_coverage.py b/tests/metrics/test_token_counter_deep_coverage.py
index 60f5a83179..8abe71a105 100644
--- a/tests/metrics/test_token_counter_deep_coverage.py
+++ b/tests/metrics/test_token_counter_deep_coverage.py
@@ -207,7 +207,8 @@ class TestOnLlmEndTokenUsage:
)
save_mock = self._run_end(cb, result)
assert cb.counts["total_tokens"] == 80
- save_mock.assert_called_once_with(50, 30)
+ save_mock.assert_called_once()
+ assert save_mock.call_args[0][:2] == (50, 30)
def test_token_usage_from_usage_metadata(self):
cb = _make_callback()
@@ -228,7 +229,8 @@ class TestOnLlmEndTokenUsage:
result = _make_llm_result(generations=[[gen]])
save_mock = self._run_end(cb, result)
assert cb.counts["total_tokens"] == 30
- save_mock.assert_called_once_with(20, 10)
+ save_mock.assert_called_once()
+ assert save_mock.call_args[0][:2] == (20, 10)
def test_token_usage_from_response_metadata_ollama(self):
cb = _make_callback()
@@ -245,14 +247,16 @@ class TestOnLlmEndTokenUsage:
result = _make_llm_result(generations=[[gen]])
save_mock = self._run_end(cb, result)
assert cb.counts["total_tokens"] == 60
- save_mock.assert_called_once_with(40, 20)
+ save_mock.assert_called_once()
+ assert save_mock.call_args[0][:2] == (40, 20)
def test_no_token_usage_saves_zero_counts(self):
"""No usage data from the provider still records the call (#4457)."""
cb = _make_callback()
result = _make_llm_result(llm_output=None, generations=[])
save_mock = self._run_end(cb, result)
- save_mock.assert_called_once_with(0, 0)
+ save_mock.assert_called_once()
+ assert save_mock.call_args[0][:2] == (0, 0)
def test_response_time_calculated(self):
cb = _make_callback()
@@ -318,7 +322,8 @@ class TestOnLlmError:
cb.on_llm_error(ValueError("bad input"))
assert cb.success_status == "error"
assert cb.error_type == "ValueError"
- mock_save.assert_called_once_with(0, 0)
+ mock_save.assert_called_once()
+ assert mock_save.call_args[0][:2] == (0, 0)
def test_calculates_response_time_on_error(self):
cb = _make_callback()
diff --git a/tests/metrics/test_token_counter_llm_paths_coverage.py b/tests/metrics/test_token_counter_llm_paths_coverage.py
index 71bde845d5..fa71849d1a 100644
--- a/tests/metrics/test_token_counter_llm_paths_coverage.py
+++ b/tests/metrics/test_token_counter_llm_paths_coverage.py
@@ -134,9 +134,10 @@ class TestOnLlmStartCallStackCapture:
assert cb.calling_file is not None
assert cb.calling_function == "execute_research"
- def test_call_stack_skips_site_packages_frames(self):
- """Frames from site-packages are skipped; if no project frame found,
- call_stack stays None."""
+ def test_call_stack_skips_third_party_frames(self):
+ """Non-project frames (langchain, third-party packages) are not
+ treated as the matched LDR frame; if no project frame is on the
+ stack, call_stack stays None."""
cb = _make_callback()
site_frame = _fake_frame(
@@ -156,6 +157,90 @@ class TestOnLlmStartCallStackCapture:
assert cb.calling_function is None
assert cb.call_stack is None
+ def test_call_stack_captures_project_frame_inside_venv(self):
+ """Regression: in the Docker image the project lives at
+ /install/.venv/.../site-packages/local_deep_research/... — the
+ 'site-packages' / 'venv' substring exclusions used to skip these
+ frames, leaving calling_file/function/call_stack as None for every
+ LLM call. The local_deep_research package-name check must match
+ them."""
+ cb = _make_callback()
+
+ project_frame = _fake_frame(
+ "/install/.venv/lib/python3.14/site-packages/local_deep_research/config/llm_config.py",
+ "invoke",
+ 362,
+ )
+ sentinel_frame = _fake_frame("/sentinel.py", "sentinel")
+
+ with patch(
+ f"{_MOD}.inspect.stack",
+ return_value=[sentinel_frame, project_frame],
+ ):
+ cb.on_llm_start({}, ["hello"])
+
+ assert cb.calling_function == "invoke"
+ assert cb.call_stack is not None
+ assert "llm_config.py:invoke:362" in cb.call_stack
+
+ def test_call_stack_includes_deep_matched_frame(self):
+ """Regression: when the matched LDR frame sits deeper than index 5
+ in the stack (common with langgraph, where llm_config.invoke wraps
+ a chain of framework frames), call_stack must still capture the
+ matched frame and any subsequent LDR frames. Previously the slice
+ was hardcoded to stack[1:6], which missed deep matches and left
+ call_stack as None for every entry."""
+ cb = _make_callback()
+
+ # Mimic a langgraph call: several framework frames, then an LDR
+ # invoke() at a deep index, then more LDR callers.
+ stack = [
+ _fake_frame("/sentinel.py", "sentinel"), # on_llm_start caller
+ _fake_frame(
+ "/install/.venv/lib/python3.14/site-packages/langchain_core/runnables/base.py",
+ "_call",
+ 100,
+ ),
+ _fake_frame(
+ "/install/.venv/lib/python3.14/site-packages/langgraph/pregel/main.py",
+ "tick",
+ 200,
+ ),
+ _fake_frame(
+ "/install/.venv/lib/python3.14/site-packages/langgraph/pregel/loop.py",
+ "run",
+ 300,
+ ),
+ _fake_frame(
+ "/install/.venv/lib/python3.14/site-packages/langgraph/utils/runnable.py",
+ "invoke",
+ 400,
+ ),
+ _fake_frame(
+ "/home/user/src/local_deep_research/config/llm_config.py",
+ "invoke",
+ 362,
+ ),
+ _fake_frame(
+ "/home/user/src/local_deep_research/advanced_search_system/strategies/langgraph_agent_strategy.py",
+ "some_method",
+ 500,
+ ),
+ ]
+
+ with patch(f"{_MOD}.inspect.stack", return_value=stack):
+ cb.on_llm_start({}, ["hello"])
+
+ assert cb.calling_function == "invoke"
+ assert cb.call_stack is not None
+ assert "llm_config.py:invoke:362" in cb.call_stack
+ assert "langgraph_agent_strategy.py:some_method:500" in cb.call_stack
+ # No third-party frames should leak in (the LDR strategy file's
+ # name happens to contain "langgraph", so check the full prefix).
+ assert "langchain_core" not in cb.call_stack
+ assert "pregel/main" not in cb.call_stack
+ assert "pregel/loop" not in cb.call_stack
+
def test_call_stack_uses_src_local_deep_research_split(self):
"""Paths containing src/local_deep_research are split on that segment."""
cb = _make_callback()
@@ -233,6 +318,174 @@ class TestOnLlmStartCallStackCapture:
assert " -> " in cb.call_stack
+class TestCallMetaRunIdIsolation:
+ """Call-stack metadata is keyed by langchain's run_id and removed on
+ on_llm_end / on_llm_error so overlapping runs and unmatched runs do
+ not inherit a previous run's stack."""
+
+ def test_meta_stored_under_run_id(self):
+ cb = _make_callback()
+ project_frame = _fake_frame(
+ "/home/user/src/local_deep_research/a.py", "f1", 1
+ )
+ sentinel = _fake_frame("/sentinel.py", "s")
+ with patch(
+ f"{_MOD}.inspect.stack", return_value=[sentinel, project_frame]
+ ):
+ cb.on_llm_start({}, ["p"], run_id="run-1")
+ assert "run-1" in cb._call_meta
+ assert cb._call_meta["run-1"]["calling_function"] == "f1"
+
+ def test_overlapping_runs_have_independent_meta(self):
+ cb = _make_callback()
+ frame_a = _fake_frame(
+ "/home/user/src/local_deep_research/a.py", "alpha", 1
+ )
+ frame_b = _fake_frame(
+ "/home/user/src/local_deep_research/b.py", "beta", 2
+ )
+ sentinel = _fake_frame("/sentinel.py", "s")
+
+ with patch(f"{_MOD}.inspect.stack", return_value=[sentinel, frame_a]):
+ cb.on_llm_start({}, ["p"], run_id="run-A")
+ with patch(f"{_MOD}.inspect.stack", return_value=[sentinel, frame_b]):
+ cb.on_llm_start({}, ["p"], run_id="run-B")
+
+ assert cb._call_meta["run-A"]["calling_function"] == "alpha"
+ assert cb._call_meta["run-B"]["calling_function"] == "beta"
+
+ def test_no_match_run_does_not_inherit_previous_stack(self):
+ """A later run that finds no LDR frame must not show the previous
+ run's stack on the saved TokenUsage row."""
+ cb = _make_callback()
+ ldr_frame = _fake_frame(
+ "/home/user/src/local_deep_research/a.py", "alpha", 1
+ )
+ non_ldr_frame = _fake_frame(
+ "/usr/lib/python3/site-packages/langchain/llms/base.py",
+ "_generate",
+ 10,
+ )
+ sentinel = _fake_frame("/sentinel.py", "s")
+
+ with patch(f"{_MOD}.inspect.stack", return_value=[sentinel, ldr_frame]):
+ cb.on_llm_start({}, ["p"], run_id="run-A")
+
+ # Second run: no project frame on the stack.
+ with patch(
+ f"{_MOD}.inspect.stack", return_value=[sentinel, non_ldr_frame]
+ ):
+ cb.on_llm_start({}, ["p"], run_id="run-B")
+
+ assert cb._call_meta["run-A"]["calling_function"] == "alpha"
+ # run-B has no LDR frame → all fields None
+ assert cb._call_meta["run-B"]["calling_file"] is None
+ assert cb._call_meta["run-B"]["calling_function"] is None
+ assert cb._call_meta["run-B"]["call_stack"] is None
+
+ def test_on_llm_end_pops_run_id_entry(self):
+ cb = _make_callback()
+ ldr_frame = _fake_frame(
+ "/home/user/src/local_deep_research/a.py", "alpha", 1
+ )
+ sentinel = _fake_frame("/sentinel.py", "s")
+ with patch(f"{_MOD}.inspect.stack", return_value=[sentinel, ldr_frame]):
+ cb.on_llm_start({}, ["p"], run_id="run-1")
+
+ with patch.object(cb, "_save_to_db") as mock_save:
+ result = MagicMock(spec=LLMResult)
+ result.llm_output = {
+ "token_usage": {
+ "prompt_tokens": 1,
+ "completion_tokens": 1,
+ "total_tokens": 2,
+ }
+ }
+ result.generations = []
+ cb.on_llm_end(result, run_id="run-1")
+
+ assert "run-1" not in cb._call_meta
+ # _save_to_db received the popped meta (alpha), not a later
+ # unrelated value.
+ assert mock_save.call_args[0][2]["calling_function"] == "alpha"
+
+ def test_on_llm_error_pops_run_id_entry(self):
+ cb = _make_callback()
+ ldr_frame = _fake_frame(
+ "/home/user/src/local_deep_research/a.py", "alpha", 1
+ )
+ sentinel = _fake_frame("/sentinel.py", "s")
+ with patch(f"{_MOD}.inspect.stack", return_value=[sentinel, ldr_frame]):
+ cb.on_llm_start({}, ["p"], run_id="run-1")
+
+ with patch.object(cb, "_save_to_db") as mock_save:
+ cb.on_llm_error(RuntimeError("boom"), run_id="run-1")
+
+ assert "run-1" not in cb._call_meta
+ assert mock_save.call_args[0][2]["calling_function"] == "alpha"
+
+ def test_unknown_run_id_in_end_yields_none_meta(self):
+ """on_llm_end for a run_id that was never registered returns
+ a None-filled meta, so we never write a previous run's stack."""
+ cb = _make_callback()
+ ldr_frame = _fake_frame(
+ "/home/user/src/local_deep_research/a.py", "alpha", 1
+ )
+ sentinel = _fake_frame("/sentinel.py", "s")
+ with patch(f"{_MOD}.inspect.stack", return_value=[sentinel, ldr_frame]):
+ cb.on_llm_start({}, ["p"], run_id="run-A")
+
+ with patch.object(cb, "_save_to_db") as mock_save:
+ result = MagicMock(spec=LLMResult)
+ result.llm_output = {
+ "token_usage": {
+ "prompt_tokens": 1,
+ "completion_tokens": 1,
+ "total_tokens": 2,
+ }
+ }
+ result.generations = []
+ # end for a different run_id
+ cb.on_llm_end(result, run_id="run-NEVER-EXISTED")
+
+ # run-A is still tracked (not popped by an unknown id)
+ assert "run-A" in cb._call_meta
+ # the saved row gets None meta, not run-A's stack
+ call_meta = mock_save.call_args[0][2]
+ assert call_meta["calling_function"] is None
+ assert call_meta["call_stack"] is None
+
+ def test_legacy_mirror_reflects_most_recent_run(self):
+ cb = _make_callback()
+ frame_a = _fake_frame(
+ "/home/user/src/local_deep_research/a.py", "alpha", 1
+ )
+ frame_b = _fake_frame(
+ "/home/user/src/local_deep_research/b.py", "beta", 2
+ )
+ sentinel = _fake_frame("/sentinel.py", "s")
+ with patch(f"{_MOD}.inspect.stack", return_value=[sentinel, frame_a]):
+ cb.on_llm_start({}, ["p"], run_id="run-A")
+ with patch(f"{_MOD}.inspect.stack", return_value=[sentinel, frame_b]):
+ cb.on_llm_start({}, ["p"], run_id="run-B")
+ # Legacy mirror follows the most recent start.
+ assert cb.calling_function == "beta"
+
+ # Pop run-B → mirror should fall back to run-A.
+ with patch.object(cb, "_save_to_db"):
+ result = MagicMock(spec=LLMResult)
+ result.llm_output = {
+ "token_usage": {
+ "prompt_tokens": 1,
+ "completion_tokens": 1,
+ "total_tokens": 2,
+ }
+ }
+ result.generations = []
+ cb.on_llm_end(result, run_id="run-B")
+ assert cb.calling_function == "alpha"
+
+
# ---------------------------------------------------------------------------
# 2. test_on_llm_start_model_from_invocation_params
# ---------------------------------------------------------------------------
@@ -396,7 +649,8 @@ class TestOnLlmEndUsageMetadataFromGenerations:
assert cb.counts["total_prompt_tokens"] == 30
assert cb.counts["total_completion_tokens"] == 15
assert cb.counts["total_tokens"] == 45
- mock_save.assert_called_once_with(30, 15)
+ mock_save.assert_called_once()
+ assert mock_save.call_args[0][:2] == (30, 15)
def test_usage_metadata_none_falls_through_to_response_metadata(self):
"""When usage_metadata is None, should continue to check response_metadata."""
@@ -412,7 +666,8 @@ class TestOnLlmEndUsageMetadataFromGenerations:
# Should pick up from response_metadata path
assert cb.counts["total_tokens"] == 30
- mock_save.assert_called_once_with(20, 10)
+ mock_save.assert_called_once()
+ assert mock_save.call_args[0][:2] == (20, 10)
def test_usage_metadata_zero_values_still_applied(self):
"""usage_metadata with zero token counts should still be applied."""
@@ -427,7 +682,8 @@ class TestOnLlmEndUsageMetadataFromGenerations:
cb.on_llm_end(result)
# token_usage dict is built but all zeros → _save_to_db called
- mock_save.assert_called_once_with(0, 0)
+ mock_save.assert_called_once()
+ assert mock_save.call_args[0][:2] == (0, 0)
def test_no_message_attribute_on_generation_no_crash(self):
"""Generation without message attribute should not crash on_llm_end."""
@@ -633,7 +889,8 @@ class TestOnLlmEndOllamaResponseMetadata:
with patch.object(cb, "_save_to_db") as mock_save:
cb.on_llm_end(result)
- mock_save.assert_called_once_with(75, 25)
+ mock_save.assert_called_once()
+ assert mock_save.call_args[0][:2] == (75, 25)
assert cb.counts["total_tokens"] == 100 # 75 + 25
def test_only_eval_count_present_triggers_branch(self):
@@ -653,7 +910,8 @@ class TestOnLlmEndOllamaResponseMetadata:
assert cb.ollama_metrics["eval_count"] == 30
assert cb.ollama_metrics["prompt_eval_count"] is None
# token_usage is built: prompt=0, completion=30, total=30
- mock_save.assert_called_once_with(0, 30)
+ mock_save.assert_called_once()
+ assert mock_save.call_args[0][:2] == (0, 30)
assert cb.counts["total_tokens"] == 30
def test_response_time_calculated_in_on_llm_end(self):
@@ -708,7 +966,8 @@ class TestOnLlmErrorTracking:
with patch.object(cb, "_save_to_db") as mock_save:
cb.on_llm_error(RuntimeError("crash"))
- mock_save.assert_called_once_with(0, 0)
+ mock_save.assert_called_once()
+ assert mock_save.call_args[0][:2] == (0, 0)
def test_response_time_calculated_when_start_time_set(self):
cb = _make_callback()
@@ -980,3 +1239,34 @@ class TestSaveToDbSuccess:
assert token_data["calling_file"] == "runner.py"
assert token_data["calling_function"] == "run_research"
assert token_data["call_stack"] == "runner.py:run_research:10"
+
+ def test_concurrent_call_meta_operations(self):
+ """Verify that concurrent additions and removals from _call_meta do not raise RuntimeError."""
+ import threading
+
+ cb = _make_callback()
+ errors = []
+
+ def worker(thread_idx):
+ try:
+ for i in range(100):
+ run_id = f"thread-{thread_idx}-run-{i}"
+ # Simulate on_llm_start
+ with patch(f"{_MOD}.inspect.stack", return_value=[]):
+ cb.on_llm_start({}, ["p"], run_id=run_id)
+ # Simulate on_llm_end
+ result = MagicMock(spec=LLMResult)
+ result.llm_output = {}
+ cb.on_llm_end(result, run_id=run_id)
+ except Exception as e:
+ errors.append(e)
+
+ threads = [
+ threading.Thread(target=worker, args=(i,)) for i in range(10)
+ ]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join()
+
+ assert not errors, f"Encountered concurrent errors: {errors}"
diff --git a/tests/metrics/test_token_counter_logic.py b/tests/metrics/test_token_counter_logic.py
index 1eb5a4e145..bd01ae9b57 100644
--- a/tests/metrics/test_token_counter_logic.py
+++ b/tests/metrics/test_token_counter_logic.py
@@ -469,7 +469,8 @@ class TestOnLLMError:
with patch.object(cb, "_save_to_db") as mock_save:
cb.on_llm_error(RuntimeError("fail"))
- mock_save.assert_called_once_with(0, 0)
+ mock_save.assert_called_once()
+ assert mock_save.call_args[0][:2] == (0, 0)
# ===========================================================================
diff --git a/tests/test_token_counter_coverage.py b/tests/test_token_counter_coverage.py
index 3a73e5aab7..c68fed8f6e 100644
--- a/tests/test_token_counter_coverage.py
+++ b/tests/test_token_counter_coverage.py
@@ -450,7 +450,15 @@ class TestOnLlmEnd:
with patch.object(cb, "_save_to_db") as mock_save:
cb.on_llm_end(_llm_result_with_token_usage(10, 20))
- mock_save.assert_called_once_with(10, 20)
+ mock_save.assert_called_once_with(
+ 10,
+ 20,
+ {
+ "calling_file": None,
+ "calling_function": None,
+ "call_stack": None,
+ },
+ )
def test_save_to_db_not_called_without_research_id(self):
cb = _make_callback() # no research_id
@@ -621,7 +629,15 @@ class TestOnLlmError:
with patch.object(cb, "_save_to_db") as mock_save:
cb.on_llm_error(RuntimeError("fail"))
- mock_save.assert_called_once_with(0, 0)
+ mock_save.assert_called_once_with(
+ 0,
+ 0,
+ {
+ "calling_file": None,
+ "calling_function": None,
+ "call_stack": None,
+ },
+ )
def test_save_to_db_not_called_on_error_without_research_id(self):
cb = _make_callback()