Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(questions): handle list-form LLM content in question generation and evidence extraction (#5559)

Ehsan2d30607be960
diff --git a/changelog.d/+question-generation-list-content.bugfix.md b/changelog.d/+question-generation-list-content.bugfix.md
new file mode 100644
index 0000000000..796d107d8f
--- /dev/null
+++ b/changelog.d/+question-generation-list-content.bugfix.md
@@ -0,0 +1 @@
+Handle list-form LLM content blocks in the question-generation family and the evidence evaluator. With a provider that returns `.content` as a list of blocks (Anthropic extended thinking or tool use), follow-up question generation, sub-question decomposition, atomic-fact decomposition, gap-filling questions, BrowseComp entity extraction and progressive searches, and evidence extraction all raised `AttributeError`. Since `StandardQuestionGenerator` is the default generator, this broke the main research loop on every iteration. These call sites now also strip `<think>` reasoning tags from model output, which is a visible change for an LLM that was not obtained through `get_llm()`.
diff --git a/src/local_deep_research/advanced_search_system/evidence/evaluator.py b/src/local_deep_research/advanced_search_system/evidence/evaluator.py
index fca4c2d3b6..693ea50667 100644
--- a/src/local_deep_research/advanced_search_system/evidence/evaluator.py
+++ b/src/local_deep_research/advanced_search_system/evidence/evaluator.py
@@ -7,6 +7,7 @@ from typing import Dict
from langchain_core.language_models import BaseChatModel
from loguru import logger
+from ...utilities.json_utils import get_llm_response_text
from ..constraints.base_constraint import Constraint
from .base_evidence import Evidence, EvidenceType
@@ -58,7 +59,7 @@ QUOTE: [relevant text]
"""
response = self.model.invoke(prompt)
- content = response.content
+ content = get_llm_response_text(response)
# Parse response
parsed = self._parse_evidence_response(content)
diff --git a/src/local_deep_research/advanced_search_system/questions/atomic_fact_question.py b/src/local_deep_research/advanced_search_system/questions/atomic_fact_question.py
index c58f387a24..f92580fe9a 100644
--- a/src/local_deep_research/advanced_search_system/questions/atomic_fact_question.py
+++ b/src/local_deep_research/advanced_search_system/questions/atomic_fact_question.py
@@ -6,6 +6,7 @@ Decomposes complex queries into atomic, independently searchable facts.
from loguru import logger
from typing import Dict, List, Optional
+from ...utilities.json_utils import get_llm_response_text
from .base_question import BaseQuestionGenerator
@@ -78,12 +79,7 @@ Where did falls occur between specific dates?
response = self.model.invoke(prompt)
- # Extract response text
- response_text = ""
- if hasattr(response, "content"):
- response_text = response.content
- else:
- response_text = str(response)
+ response_text = get_llm_response_text(response)
# Parse questions
questions = []
@@ -148,12 +144,7 @@ Return ONLY the questions, one per line.
response = self.model.invoke(prompt)
- # Extract response text
- response_text = ""
- if hasattr(response, "content"):
- response_text = response.content
- else:
- response_text = str(response)
+ response_text = get_llm_response_text(response)
# Parse questions
questions = []
diff --git a/src/local_deep_research/advanced_search_system/questions/browsecomp_question.py b/src/local_deep_research/advanced_search_system/questions/browsecomp_question.py
index a515150c33..34935ca48e 100644
--- a/src/local_deep_research/advanced_search_system/questions/browsecomp_question.py
+++ b/src/local_deep_research/advanced_search_system/questions/browsecomp_question.py
@@ -7,6 +7,7 @@ from typing import Dict, List, Optional
from loguru import logger
+from ...utilities.json_utils import get_llm_response_text
from .base_question import BaseQuestionGenerator
@@ -94,9 +95,7 @@ DESCRIPTORS: [entity1], [entity2], ...
"""
response = self.model.invoke(prompt)
- content = (
- response.content if hasattr(response, "content") else str(response)
- )
+ content = get_llm_response_text(response)
entities: Dict[str, List[str]] = {
"temporal": [],
@@ -280,9 +279,7 @@ Format: One search per line
"""
response = self.model.invoke(prompt)
- content = (
- response.content if hasattr(response, "content") else str(response)
- )
+ content = get_llm_response_text(response)
# Extract searches from response
searches = []
diff --git a/src/local_deep_research/advanced_search_system/questions/flexible_browsecomp_question.py b/src/local_deep_research/advanced_search_system/questions/flexible_browsecomp_question.py
index 93368fe2dd..67125ff071 100644
--- a/src/local_deep_research/advanced_search_system/questions/flexible_browsecomp_question.py
+++ b/src/local_deep_research/advanced_search_system/questions/flexible_browsecomp_question.py
@@ -6,6 +6,7 @@ Inherits from BrowseComp but gives the LLM more freedom in search strategy.
from typing import Dict, List
+from ...utilities.json_utils import get_llm_response_text
from .browsecomp_question import BrowseCompQuestionGenerator
@@ -59,9 +60,7 @@ Create {num_questions} diverse search queries. Avoid exact duplicates. One per l
"""
response = self.model.invoke(prompt)
- content = (
- response.content if hasattr(response, "content") else str(response)
- )
+ content = get_llm_response_text(response)
# Extract searches
searches = []
diff --git a/src/local_deep_research/advanced_search_system/questions/standard_question.py b/src/local_deep_research/advanced_search_system/questions/standard_question.py
index d254c3c04e..a983e4dd0d 100644
--- a/src/local_deep_research/advanced_search_system/questions/standard_question.py
+++ b/src/local_deep_research/advanced_search_system/questions/standard_question.py
@@ -7,6 +7,7 @@ from typing import List, Optional
from loguru import logger
+from ...utilities.json_utils import get_llm_response_text
from .base_question import BaseQuestionGenerator
@@ -40,13 +41,7 @@ class StandardQuestionGenerator(BaseQuestionGenerator):
response = self.model.invoke(prompt)
- # Handle both string responses and responses with .content attribute
- response_text = ""
- if hasattr(response, "content"):
- response_text = response.content
- else:
- # Handle string responses
- response_text = str(response)
+ response_text = get_llm_response_text(response)
questions = [
q.replace("Q:", "").strip()
@@ -94,13 +89,7 @@ Only provide the numbered sub-questions, nothing else."""
try:
response = self.model.invoke(prompt)
- # Handle both string responses and responses with .content attribute
- content = ""
- if hasattr(response, "content"):
- content = response.content
- else:
- # Handle string responses
- content = str(response)
+ content = get_llm_response_text(response)
# Parse sub-questions from the response
sub_questions = []
diff --git a/tests/advanced_search_system/evidence/test_evaluator_list_content.py b/tests/advanced_search_system/evidence/test_evaluator_list_content.py
new file mode 100644
index 0000000000..9b8fcec233
--- /dev/null
+++ b/tests/advanced_search_system/evidence/test_evaluator_list_content.py
@@ -0,0 +1,56 @@
+"""
+Regression test for list-form LLM content in the evidence evaluator.
+
+evaluator.py read ``response.content`` directly and handed it to
+``_parse_evidence_response``, which calls ``.strip().split()`` on it. List-form
+content blocks (Anthropic extended-thinking or tool-use) raised AttributeError
+there. The fix routes the read through ``get_llm_response_text``.
+
+Sibling of tests/advanced_search_system/questions/test_question_generation_list_content.py.
+"""
+
+from unittest.mock import Mock
+
+
+class TestExtractEvidenceListContent:
+ """extract_evidence must parse evidence out of list-form content."""
+
+ def test_extract_evidence_parses_list_content(self):
+ """The parsed claim and confidence come from the text block."""
+ from local_deep_research.advanced_search_system.evidence.evaluator import (
+ EvidenceEvaluator,
+ )
+ from local_deep_research.advanced_search_system.constraints.base_constraint import (
+ Constraint,
+ ConstraintType,
+ )
+
+ model = Mock()
+ model.invoke.return_value = Mock(
+ content=[
+ {
+ "type": "text",
+ "text": (
+ "CLAIM: The bridge opened in 1999\n"
+ "TYPE: official_record\n"
+ "SOURCE: City archive\n"
+ "CONFIDENCE: 0.9"
+ ),
+ }
+ ]
+ )
+ constraint = Constraint(
+ id="test",
+ type=ConstraintType.PROPERTY,
+ value="1999",
+ description="Opened in 1999",
+ )
+
+ evidence = EvidenceEvaluator(model).extract_evidence(
+ search_result="the bridge opened in 1999",
+ candidate="the bridge",
+ constraint=constraint,
+ )
+
+ assert evidence.claim == "The bridge opened in 1999"
+ assert evidence.confidence == 0.9
diff --git a/tests/advanced_search_system/questions/test_question_generation_list_content.py b/tests/advanced_search_system/questions/test_question_generation_list_content.py
new file mode 100644
index 0000000000..37811733c4
--- /dev/null
+++ b/tests/advanced_search_system/questions/test_question_generation_list_content.py
@@ -0,0 +1,207 @@
+"""
+Regression tests for list-form LLM content in the question-generation family.
+
+These seven call sites read ``response.content`` directly and then apply string
+methods to it. When a provider returns content as a LIST of blocks (Anthropic
+extended-thinking or tool-use) that raises AttributeError, so the default
+research loop fails on every iteration. The fix routes each read through
+``get_llm_response_text``, which extracts the text from the blocks.
+
+Same shape as tests/advanced_search_system/test_list_content_coercion.py, which
+covers the sites fixed in #5422.
+"""
+
+from unittest.mock import Mock
+
+
+def _list_content_response(text):
+ """Build a fake LLM response with Anthropic-style list content blocks."""
+ response = Mock()
+ response.content = [{"type": "text", "text": text}]
+ return response
+
+
+def _entities():
+ """Build the entity dict shape _extract_entities returns."""
+ return {
+ "temporal": ["1999"],
+ "numerical": [],
+ "names": ["Ada Lovelace"],
+ "locations": ["Paris"],
+ "descriptors": [],
+ }
+
+
+class TestStandardQuestionGenerator:
+ """standard_question.py — the DEFAULT question generator."""
+
+ def test_generate_questions_parses_list_content(self):
+ """generate_questions must read Q: lines out of list-form content."""
+ from local_deep_research.advanced_search_system.questions.standard_question import (
+ StandardQuestionGenerator,
+ )
+
+ model = Mock()
+ model.invoke.return_value = _list_content_response(
+ "Q: What is the capital?\nQ: When was it founded?"
+ )
+
+ questions = StandardQuestionGenerator(model).generate_questions(
+ current_knowledge="none", query="a city", questions_per_iteration=2
+ )
+
+ assert questions == ["What is the capital?", "When was it founded?"]
+
+ def test_generate_sub_questions_parses_list_content(self):
+ """generate_sub_questions must read numbered lines out of list content."""
+ from local_deep_research.advanced_search_system.questions.standard_question import (
+ StandardQuestionGenerator,
+ )
+
+ model = Mock()
+ model.invoke.return_value = _list_content_response(
+ "1. What is the capital?\n2. When was it founded?"
+ )
+
+ sub_questions = StandardQuestionGenerator(model).generate_sub_questions(
+ query="a city"
+ )
+
+ assert sub_questions == [
+ "What is the capital?",
+ "When was it founded?",
+ ]
+
+
+class TestAtomicFactQuestionGenerator:
+ """atomic_fact_question.py — both LLM reads."""
+
+ def test_decompose_to_atomic_facts_parses_list_content(self):
+ """_decompose_to_atomic_facts must split list-form content into facts."""
+ from local_deep_research.advanced_search_system.questions.atomic_fact_question import (
+ AtomicFactQuestionGenerator,
+ )
+
+ model = Mock()
+ model.invoke.return_value = _list_content_response(
+ "1. Which river runs through the city?\n"
+ "2. Which year was the bridge completed?"
+ )
+
+ facts = AtomicFactQuestionGenerator(model)._decompose_to_atomic_facts(
+ "a city on a river"
+ )
+
+ assert facts == [
+ "Which river runs through the city?",
+ "Which year was the bridge completed?",
+ ]
+
+ def test_gap_filling_questions_parse_list_content(self):
+ """_generate_gap_filling_questions must split list-form content."""
+ from local_deep_research.advanced_search_system.questions.atomic_fact_question import (
+ AtomicFactQuestionGenerator,
+ )
+
+ model = Mock()
+ model.invoke.return_value = _list_content_response(
+ "1. Which river runs through the city?\n"
+ "2. Which year was the bridge completed?"
+ )
+
+ questions = AtomicFactQuestionGenerator(
+ model
+ )._generate_gap_filling_questions(
+ original_query="a city on a river",
+ current_knowledge="none",
+ questions_by_iteration={},
+ questions_per_iteration=2,
+ )
+
+ assert questions == [
+ "Which river runs through the city?",
+ "Which year was the bridge completed?",
+ ]
+
+
+class TestBrowseCompQuestionGenerator:
+ """browsecomp_question.py — entity extraction and progressive searches."""
+
+ def test_extract_entities_parses_list_content(self):
+ """_extract_entities must read its categories out of list content."""
+ from local_deep_research.advanced_search_system.questions.browsecomp_question import (
+ BrowseCompQuestionGenerator,
+ )
+
+ model = Mock()
+ model.invoke.return_value = _list_content_response(
+ "TEMPORAL: 1999\nNAMES: Ada Lovelace\nLOCATIONS: Paris"
+ )
+
+ entities = BrowseCompQuestionGenerator(model)._extract_entities(
+ "who was where in 1999"
+ )
+
+ assert entities["names"] == ["Ada Lovelace"]
+ assert entities["locations"] == ["Paris"]
+ assert "1999" in entities["temporal"]
+
+ def test_progressive_searches_parse_list_content(self):
+ """_generate_progressive_searches must split list-form content."""
+ from local_deep_research.advanced_search_system.questions.browsecomp_question import (
+ BrowseCompQuestionGenerator,
+ )
+
+ model = Mock()
+ model.invoke.return_value = _list_content_response(
+ "Ada Lovelace Paris 1999\nAda Lovelace bridge completion"
+ )
+
+ searches = BrowseCompQuestionGenerator(
+ model
+ )._generate_progressive_searches(
+ query="who was where in 1999",
+ current_knowledge="none",
+ entities=_entities(),
+ questions_by_iteration={},
+ results_by_iteration={},
+ num_questions=2,
+ iteration=1,
+ )
+
+ assert searches == [
+ "Ada Lovelace Paris 1999",
+ "Ada Lovelace bridge completion",
+ ]
+
+
+class TestFlexibleBrowseCompQuestionGenerator:
+ """flexible_browsecomp_question.py — the variant's own LLM read."""
+
+ def test_progressive_searches_parse_list_content(self):
+ """_generate_progressive_searches must split list-form content."""
+ from local_deep_research.advanced_search_system.questions.flexible_browsecomp_question import (
+ FlexibleBrowseCompQuestionGenerator,
+ )
+
+ model = Mock()
+ model.invoke.return_value = _list_content_response(
+ "Ada Lovelace Paris 1999\nAda Lovelace bridge completion"
+ )
+
+ searches = FlexibleBrowseCompQuestionGenerator(
+ model
+ )._generate_progressive_searches(
+ query="who was where in 1999",
+ current_knowledge="none",
+ entities=_entities(),
+ questions_by_iteration={},
+ results_by_iteration={},
+ num_questions=2,
+ iteration=1,
+ )
+
+ assert searches == [
+ "Ada Lovelace Paris 1999",
+ "Ada Lovelace bridge completion",
+ ]