~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(questions): handle list-form LLM content in question generation and evidence extraction (#5559)
Ehsan2d30607be960
diff --git a/changelog.d/+question-generation-list-content.bugfix.md b/changelog.d/+question-generation-list-content.bugfix.mdnew file mode 100644index 0000000000..796d107d8f--- /dev/null+++ b/changelog.d/+question-generation-list-content.bugfix.md@@ -0,0 +1 @@+Handle list-form LLM content blocks in the question-generation family and the evidence evaluator. With a provider that returns `.content` as a list of blocks (Anthropic extended thinking or tool use), follow-up question generation, sub-question decomposition, atomic-fact decomposition, gap-filling questions, BrowseComp entity extraction and progressive searches, and evidence extraction all raised `AttributeError`. Since `StandardQuestionGenerator` is the default generator, this broke the main research loop on every iteration. These call sites now also strip `<think>` reasoning tags from model output, which is a visible change for an LLM that was not obtained through `get_llm()`.diff --git a/src/local_deep_research/advanced_search_system/evidence/evaluator.py b/src/local_deep_research/advanced_search_system/evidence/evaluator.pyindex fca4c2d3b6..693ea50667 100644--- a/src/local_deep_research/advanced_search_system/evidence/evaluator.py+++ b/src/local_deep_research/advanced_search_system/evidence/evaluator.py@@ -7,6 +7,7 @@ from typing import Dictfrom langchain_core.language_models import BaseChatModelfrom loguru import logger+from ...utilities.json_utils import get_llm_response_textfrom ..constraints.base_constraint import Constraintfrom .base_evidence import Evidence, EvidenceType@@ -58,7 +59,7 @@ QUOTE: [relevant text]"""response = self.model.invoke(prompt)- content = response.content+ content = get_llm_response_text(response)# Parse responseparsed = self._parse_evidence_response(content)diff --git a/src/local_deep_research/advanced_search_system/questions/atomic_fact_question.py b/src/local_deep_research/advanced_search_system/questions/atomic_fact_question.pyindex c58f387a24..f92580fe9a 100644--- a/src/local_deep_research/advanced_search_system/questions/atomic_fact_question.py+++ b/src/local_deep_research/advanced_search_system/questions/atomic_fact_question.py@@ -6,6 +6,7 @@ Decomposes complex queries into atomic, independently searchable facts.from loguru import loggerfrom typing import Dict, List, Optional+from ...utilities.json_utils import get_llm_response_textfrom .base_question import BaseQuestionGenerator@@ -78,12 +79,7 @@ Where did falls occur between specific dates?response = self.model.invoke(prompt)- # Extract response text- response_text = ""- if hasattr(response, "content"):- response_text = response.content- else:- response_text = str(response)+ response_text = get_llm_response_text(response)# Parse questionsquestions = []@@ -148,12 +144,7 @@ Return ONLY the questions, one per line.response = self.model.invoke(prompt)- # Extract response text- response_text = ""- if hasattr(response, "content"):- response_text = response.content- else:- response_text = str(response)+ response_text = get_llm_response_text(response)# Parse questionsquestions = []diff --git a/src/local_deep_research/advanced_search_system/questions/browsecomp_question.py b/src/local_deep_research/advanced_search_system/questions/browsecomp_question.pyindex a515150c33..34935ca48e 100644--- a/src/local_deep_research/advanced_search_system/questions/browsecomp_question.py+++ b/src/local_deep_research/advanced_search_system/questions/browsecomp_question.py@@ -7,6 +7,7 @@ from typing import Dict, List, Optionalfrom loguru import logger+from ...utilities.json_utils import get_llm_response_textfrom .base_question import BaseQuestionGenerator@@ -94,9 +95,7 @@ DESCRIPTORS: [entity1], [entity2], ..."""response = self.model.invoke(prompt)- content = (- response.content if hasattr(response, "content") else str(response)- )+ content = get_llm_response_text(response)entities: Dict[str, List[str]] = {"temporal": [],@@ -280,9 +279,7 @@ Format: One search per line"""response = self.model.invoke(prompt)- content = (- response.content if hasattr(response, "content") else str(response)- )+ content = get_llm_response_text(response)# Extract searches from responsesearches = []diff --git a/src/local_deep_research/advanced_search_system/questions/flexible_browsecomp_question.py b/src/local_deep_research/advanced_search_system/questions/flexible_browsecomp_question.pyindex 93368fe2dd..67125ff071 100644--- a/src/local_deep_research/advanced_search_system/questions/flexible_browsecomp_question.py+++ b/src/local_deep_research/advanced_search_system/questions/flexible_browsecomp_question.py@@ -6,6 +6,7 @@ Inherits from BrowseComp but gives the LLM more freedom in search strategy.from typing import Dict, List+from ...utilities.json_utils import get_llm_response_textfrom .browsecomp_question import BrowseCompQuestionGenerator@@ -59,9 +60,7 @@ Create {num_questions} diverse search queries. Avoid exact duplicates. One per l"""response = self.model.invoke(prompt)- content = (- response.content if hasattr(response, "content") else str(response)- )+ content = get_llm_response_text(response)# Extract searchessearches = []diff --git a/src/local_deep_research/advanced_search_system/questions/standard_question.py b/src/local_deep_research/advanced_search_system/questions/standard_question.pyindex d254c3c04e..a983e4dd0d 100644--- a/src/local_deep_research/advanced_search_system/questions/standard_question.py+++ b/src/local_deep_research/advanced_search_system/questions/standard_question.py@@ -7,6 +7,7 @@ from typing import List, Optionalfrom loguru import logger+from ...utilities.json_utils import get_llm_response_textfrom .base_question import BaseQuestionGenerator@@ -40,13 +41,7 @@ class StandardQuestionGenerator(BaseQuestionGenerator):response = self.model.invoke(prompt)- # Handle both string responses and responses with .content attribute- response_text = ""- if hasattr(response, "content"):- response_text = response.content- else:- # Handle string responses- response_text = str(response)+ response_text = get_llm_response_text(response)questions = [q.replace("Q:", "").strip()@@ -94,13 +89,7 @@ Only provide the numbered sub-questions, nothing else."""try:response = self.model.invoke(prompt)- # Handle both string responses and responses with .content attribute- content = ""- if hasattr(response, "content"):- content = response.content- else:- # Handle string responses- content = str(response)+ content = get_llm_response_text(response)# Parse sub-questions from the responsesub_questions = []diff --git a/tests/advanced_search_system/evidence/test_evaluator_list_content.py b/tests/advanced_search_system/evidence/test_evaluator_list_content.pynew file mode 100644index 0000000000..9b8fcec233--- /dev/null+++ b/tests/advanced_search_system/evidence/test_evaluator_list_content.py@@ -0,0 +1,56 @@+"""+Regression test for list-form LLM content in the evidence evaluator.++evaluator.py read ``response.content`` directly and handed it to+``_parse_evidence_response``, which calls ``.strip().split()`` on it. List-form+content blocks (Anthropic extended-thinking or tool-use) raised AttributeError+there. The fix routes the read through ``get_llm_response_text``.++Sibling of tests/advanced_search_system/questions/test_question_generation_list_content.py.+"""++from unittest.mock import Mock+++class TestExtractEvidenceListContent:+ """extract_evidence must parse evidence out of list-form content."""++ def test_extract_evidence_parses_list_content(self):+ """The parsed claim and confidence come from the text block."""+ from local_deep_research.advanced_search_system.evidence.evaluator import (+ EvidenceEvaluator,+ )+ from local_deep_research.advanced_search_system.constraints.base_constraint import (+ Constraint,+ ConstraintType,+ )++ model = Mock()+ model.invoke.return_value = Mock(+ content=[+ {+ "type": "text",+ "text": (+ "CLAIM: The bridge opened in 1999\n"+ "TYPE: official_record\n"+ "SOURCE: City archive\n"+ "CONFIDENCE: 0.9"+ ),+ }+ ]+ )+ constraint = Constraint(+ id="test",+ type=ConstraintType.PROPERTY,+ value="1999",+ description="Opened in 1999",+ )++ evidence = EvidenceEvaluator(model).extract_evidence(+ search_result="the bridge opened in 1999",+ candidate="the bridge",+ constraint=constraint,+ )++ assert evidence.claim == "The bridge opened in 1999"+ assert evidence.confidence == 0.9diff --git a/tests/advanced_search_system/questions/test_question_generation_list_content.py b/tests/advanced_search_system/questions/test_question_generation_list_content.pynew file mode 100644index 0000000000..37811733c4--- /dev/null+++ b/tests/advanced_search_system/questions/test_question_generation_list_content.py@@ -0,0 +1,207 @@+"""+Regression tests for list-form LLM content in the question-generation family.++These seven call sites read ``response.content`` directly and then apply string+methods to it. When a provider returns content as a LIST of blocks (Anthropic+extended-thinking or tool-use) that raises AttributeError, so the default+research loop fails on every iteration. The fix routes each read through+``get_llm_response_text``, which extracts the text from the blocks.++Same shape as tests/advanced_search_system/test_list_content_coercion.py, which+covers the sites fixed in #5422.+"""++from unittest.mock import Mock+++def _list_content_response(text):+ """Build a fake LLM response with Anthropic-style list content blocks."""+ response = Mock()+ response.content = [{"type": "text", "text": text}]+ return response+++def _entities():+ """Build the entity dict shape _extract_entities returns."""+ return {+ "temporal": ["1999"],+ "numerical": [],+ "names": ["Ada Lovelace"],+ "locations": ["Paris"],+ "descriptors": [],+ }+++class TestStandardQuestionGenerator:+ """standard_question.py — the DEFAULT question generator."""++ def test_generate_questions_parses_list_content(self):+ """generate_questions must read Q: lines out of list-form content."""+ from local_deep_research.advanced_search_system.questions.standard_question import (+ StandardQuestionGenerator,+ )++ model = Mock()+ model.invoke.return_value = _list_content_response(+ "Q: What is the capital?\nQ: When was it founded?"+ )++ questions = StandardQuestionGenerator(model).generate_questions(+ current_knowledge="none", query="a city", questions_per_iteration=2+ )++ assert questions == ["What is the capital?", "When was it founded?"]++ def test_generate_sub_questions_parses_list_content(self):+ """generate_sub_questions must read numbered lines out of list content."""+ from local_deep_research.advanced_search_system.questions.standard_question import (+ StandardQuestionGenerator,+ )++ model = Mock()+ model.invoke.return_value = _list_content_response(+ "1. What is the capital?\n2. When was it founded?"+ )++ sub_questions = StandardQuestionGenerator(model).generate_sub_questions(+ query="a city"+ )++ assert sub_questions == [+ "What is the capital?",+ "When was it founded?",+ ]+++class TestAtomicFactQuestionGenerator:+ """atomic_fact_question.py — both LLM reads."""++ def test_decompose_to_atomic_facts_parses_list_content(self):+ """_decompose_to_atomic_facts must split list-form content into facts."""+ from local_deep_research.advanced_search_system.questions.atomic_fact_question import (+ AtomicFactQuestionGenerator,+ )++ model = Mock()+ model.invoke.return_value = _list_content_response(+ "1. Which river runs through the city?\n"+ "2. Which year was the bridge completed?"+ )++ facts = AtomicFactQuestionGenerator(model)._decompose_to_atomic_facts(+ "a city on a river"+ )++ assert facts == [+ "Which river runs through the city?",+ "Which year was the bridge completed?",+ ]++ def test_gap_filling_questions_parse_list_content(self):+ """_generate_gap_filling_questions must split list-form content."""+ from local_deep_research.advanced_search_system.questions.atomic_fact_question import (+ AtomicFactQuestionGenerator,+ )++ model = Mock()+ model.invoke.return_value = _list_content_response(+ "1. Which river runs through the city?\n"+ "2. Which year was the bridge completed?"+ )++ questions = AtomicFactQuestionGenerator(+ model+ )._generate_gap_filling_questions(+ original_query="a city on a river",+ current_knowledge="none",+ questions_by_iteration={},+ questions_per_iteration=2,+ )++ assert questions == [+ "Which river runs through the city?",+ "Which year was the bridge completed?",+ ]+++class TestBrowseCompQuestionGenerator:+ """browsecomp_question.py — entity extraction and progressive searches."""++ def test_extract_entities_parses_list_content(self):+ """_extract_entities must read its categories out of list content."""+ from local_deep_research.advanced_search_system.questions.browsecomp_question import (+ BrowseCompQuestionGenerator,+ )++ model = Mock()+ model.invoke.return_value = _list_content_response(+ "TEMPORAL: 1999\nNAMES: Ada Lovelace\nLOCATIONS: Paris"+ )++ entities = BrowseCompQuestionGenerator(model)._extract_entities(+ "who was where in 1999"+ )++ assert entities["names"] == ["Ada Lovelace"]+ assert entities["locations"] == ["Paris"]+ assert "1999" in entities["temporal"]++ def test_progressive_searches_parse_list_content(self):+ """_generate_progressive_searches must split list-form content."""+ from local_deep_research.advanced_search_system.questions.browsecomp_question import (+ BrowseCompQuestionGenerator,+ )++ model = Mock()+ model.invoke.return_value = _list_content_response(+ "Ada Lovelace Paris 1999\nAda Lovelace bridge completion"+ )++ searches = BrowseCompQuestionGenerator(+ model+ )._generate_progressive_searches(+ query="who was where in 1999",+ current_knowledge="none",+ entities=_entities(),+ questions_by_iteration={},+ results_by_iteration={},+ num_questions=2,+ iteration=1,+ )++ assert searches == [+ "Ada Lovelace Paris 1999",+ "Ada Lovelace bridge completion",+ ]+++class TestFlexibleBrowseCompQuestionGenerator:+ """flexible_browsecomp_question.py — the variant's own LLM read."""++ def test_progressive_searches_parse_list_content(self):+ """_generate_progressive_searches must split list-form content."""+ from local_deep_research.advanced_search_system.questions.flexible_browsecomp_question import (+ FlexibleBrowseCompQuestionGenerator,+ )++ model = Mock()+ model.invoke.return_value = _list_content_response(+ "Ada Lovelace Paris 1999\nAda Lovelace bridge completion"+ )++ searches = FlexibleBrowseCompQuestionGenerator(+ model+ )._generate_progressive_searches(+ query="who was where in 1999",+ current_knowledge="none",+ entities=_entities(),+ questions_by_iteration={},+ results_by_iteration={},+ num_questions=2,+ iteration=1,+ )++ assert searches == [+ "Ada Lovelace Paris 1999",+ "Ada Lovelace bridge completion",+ ]