Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(research): refuse LLM synthesis when no sources are available (prevents fabricated citations) (#4503)

LearningCircuit203cb6f83ae2
diff --git a/changelog.d/4503.bugfix.md b/changelog.d/4503.bugfix.md
new file mode 100644
index 0000000000..227a8a8885
--- /dev/null
+++ b/changelog.d/4503.bugfix.md
@@ -0,0 +1 @@
+Research no longer fabricates citations when every search returns nothing: if no sources are found (or a local collection search fails), the answer now states this explicitly instead of inventing references, the progress log shows an error, and failed collection searches are reported instead of being silently treated as "no matching documents".
diff --git a/src/local_deep_research/advanced_search_system/strategies/source_based_strategy.py b/src/local_deep_research/advanced_search_system/strategies/source_based_strategy.py
index cbb718112e..7d3fd57dfd 100644
--- a/src/local_deep_research/advanced_search_system/strategies/source_based_strategy.py
+++ b/src/local_deep_research/advanced_search_system/strategies/source_based_strategy.py
@@ -459,11 +459,22 @@ class SourceBasedSearchStrategy(BaseSearchStrategy):
# nr_of_links offsets these numbers so they continue from previous
# analyze_topic() calls (critical for detailed report mode where
# multiple subsections each add their own citations).
- self._update_progress(
- f"Synthesizing {len(final_filtered_results)} sources from {iterations_to_run} iterations...",
- 90,
- {"phase": "synthesis", "type": "milestone"},
- )
+ if final_filtered_results:
+ self._update_progress(
+ f"Synthesizing {len(final_filtered_results)} sources from {iterations_to_run} iterations...",
+ 90,
+ {"phase": "synthesis", "type": "milestone"},
+ )
+ else:
+ # All searches returned nothing (no matches, or engine
+ # errors — see logs). The citation handler refuses to
+ # invoke the LLM without sources (it would fabricate
+ # citations) and returns an explicit no-sources message.
+ self._update_progress(
+ "No sources found — answer cannot be generated (check logs for search errors)",
+ 90,
+ {"phase": "synthesis", "type": "error"},
+ )
final_citation_result = self.citation_handler.analyze_followup(
query,
diff --git a/src/local_deep_research/citation_handlers/base_citation_handler.py b/src/local_deep_research/citation_handlers/base_citation_handler.py
index b8ef92b107..6fd9117bda 100644
--- a/src/local_deep_research/citation_handlers/base_citation_handler.py
+++ b/src/local_deep_research/citation_handlers/base_citation_handler.py
@@ -180,6 +180,30 @@ class BaseCitationHandler(ABC):
sources.append(f"[{source_id}] {doc.page_content}")
return "\n\n".join(sources)
+ def _no_sources_response(self, question: str) -> Dict[str, Any]:
+ """
+ Explicit no-sources result returned instead of invoking the LLM.
+
+ Prompting the LLM to "answer with citations [1], [2]…" while the
+ sources section is empty makes it fall back on its training data
+ and fabricate references. Handlers call this to refuse synthesis
+ instead.
+ """
+ logger.warning(
+ f"[{type(self).__name__}] No sources available for synthesis of "
+ f"'{question[:100]}' — skipping LLM call to avoid fabricated "
+ f"citations"
+ )
+ content = (
+ "No sources were found for this question. The selected search "
+ "engines or document collections returned no results; this can "
+ "also happen when a search fails with an error (check the "
+ "research logs). No answer was generated because, without "
+ "sources, it would have to rely on the language model's "
+ "built-in knowledge and could contain fabricated citations."
+ )
+ return {"content": content, "documents": []}
+
@abstractmethod
def analyze_initial(
self, query: str, search_results: Union[str, List[Dict]]
diff --git a/src/local_deep_research/citation_handlers/standard_citation_handler.py b/src/local_deep_research/citation_handlers/standard_citation_handler.py
index 03710480b1..3b59cd2445 100644
--- a/src/local_deep_research/citation_handlers/standard_citation_handler.py
+++ b/src/local_deep_research/citation_handlers/standard_citation_handler.py
@@ -15,6 +15,8 @@ class StandardCitationHandler(BaseCitationHandler):
self, query: str, search_results: Union[str, List[Dict]]
) -> Dict[str, Any]:
documents = self._create_documents(search_results)
+ if not documents:
+ return self._no_sources_response(query)
formatted_sources = self._format_sources(documents)
current_timestamp = datetime.now(timezone.utc).strftime(
"%Y-%m-%d %H:%M"
@@ -48,6 +50,10 @@ Provide a detailed analysis with citations. Do not create the bibliography, it w
documents = self._create_documents(
search_results, nr_of_links=nr_of_links
)
+ # With previous knowledge present the LLM can still cite prior
+ # sources legitimately; with neither, refuse to synthesize.
+ if not documents and not (previous_knowledge or "").strip():
+ return self._no_sources_response(question)
formatted_sources = self._format_sources(documents)
# Add fact-checking step
fact_check_prompt = f"""Analyze these sources for factual consistency:
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_collection.py b/src/local_deep_research/web_search_engines/engines/search_engine_collection.py
index 2dd55e10ce..4d0b9ef1d8 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_collection.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_collection.py
@@ -241,10 +241,13 @@ class CollectionSearchEngine(LibraryRAGSearchEngine):
return results
except Exception:
+ # Re-raise instead of returning [] so a failed search is not
+ # indistinguishable from "no matching documents": run() records
+ # the failure in metrics, and API callers can report the error.
logger.exception(
f"Error searching collection '{self.collection_name}'"
)
- return []
+ raise
def _get_document_url(self, doc_id: Optional[str]) -> str:
"""Get the URL for viewing a document."""
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_library.py b/src/local_deep_research/web_search_engines/engines/search_engine_library.py
index 9f806896bc..fb4455e477 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_library.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_library.py
@@ -118,6 +118,7 @@ class LibraryRAGSearchEngine(BaseSearchEngine):
# Search across all collections and merge results
all_docs_with_scores = []
+ failed_collections = []
for collection in collections:
collection_id = collection.get("id")
if not collection_id:
@@ -191,9 +192,13 @@ class LibraryRAGSearchEngine(BaseSearchEngine):
all_docs_with_scores.append((doc, score))
except Exception:
- logger.warning(
+ # One broken collection must not abort the others, but
+ # record the failure so it is not silently equated with
+ # "no matching documents" below.
+ logger.exception(
f"Error searching collection {collection_id}"
)
+ failed_collections.append(collection_id)
continue
# Sort all results by score (lower is better for distance)
@@ -203,6 +208,10 @@ class LibraryRAGSearchEngine(BaseSearchEngine):
docs_with_scores = all_docs_with_scores[:limit]
if not docs_with_scores:
+ if failed_collections:
+ # Zero results AND at least one collection errored:
+ # reporting "no results" here would hide the failure.
+ self._raise_collections_failed(failed_collections)
logger.info("No results found across any collections")
return []
@@ -286,8 +295,18 @@ class LibraryRAGSearchEngine(BaseSearchEngine):
return results
except Exception:
+ # Re-raise so run() records the failure in metrics instead of
+ # treating a failed search as "no matching documents".
logger.exception("Error searching library RAG")
- return []
+ raise
+
+ @staticmethod
+ def _raise_collections_failed(failed_collections: List[str]) -> None:
+ """Signal zero results caused (at least partly) by search failures."""
+ raise RuntimeError(
+ f"Library search returned no results and failed for "
+ f"{len(failed_collections)} collection(s): {failed_collections}"
+ )
def _get_previews(
self,
diff --git a/tests/citation_handlers/test_citation_handler_edge_cases.py b/tests/citation_handlers/test_citation_handler_edge_cases.py
index 1957b2d2c6..0afcabea09 100644
--- a/tests/citation_handlers/test_citation_handler_edge_cases.py
+++ b/tests/citation_handlers/test_citation_handler_edge_cases.py
@@ -205,7 +205,10 @@ class TestStandardCitationHandlerAnalyzeInitial:
mock_llm.invoke.return_value = MagicMock(content="response")
handler = StandardCitationHandler(llm=mock_llm)
- handler.analyze_initial("what is quantum computing", [])
+ results = [
+ {"full_content": "content", "link": "http://a.com", "title": "A"},
+ ]
+ handler.analyze_initial("what is quantum computing", results)
prompt = mock_llm.invoke.call_args[0][0]
assert "what is quantum computing" in prompt
diff --git a/tests/citation_handlers/test_no_sources_guard.py b/tests/citation_handlers/test_no_sources_guard.py
new file mode 100644
index 0000000000..701cd97dce
--- /dev/null
+++ b/tests/citation_handlers/test_no_sources_guard.py
@@ -0,0 +1,106 @@
+"""
+Tests for the no-sources guard in StandardCitationHandler.
+
+When all searches return nothing (no matches or engine errors), the
+handler must NOT invoke the LLM: prompting for a cited answer with an
+empty sources section makes the model fabricate references from its
+training data (e.g. invented blog posts). See #4480.
+"""
+
+from unittest.mock import MagicMock
+
+
+def _make_handler(settings_snapshot=None):
+ from local_deep_research.citation_handlers.standard_citation_handler import (
+ StandardCitationHandler,
+ )
+
+ mock_llm = MagicMock()
+ mock_llm.invoke.return_value = MagicMock(content="response [1]")
+ return StandardCitationHandler(
+ llm=mock_llm, settings_snapshot=settings_snapshot or {}
+ ), mock_llm
+
+
+class TestAnalyzeInitialNoSourcesGuard:
+ def test_empty_results_skip_llm(self):
+ """No documents -> no LLM call, explicit no-sources content."""
+ handler, mock_llm = _make_handler()
+
+ result = handler.analyze_initial("test query", [])
+
+ mock_llm.invoke.assert_not_called()
+ assert result["documents"] == []
+ assert "No sources were found" in result["content"]
+
+ def test_string_results_skip_llm(self):
+ """String input produces no documents -> same guard applies."""
+ handler, mock_llm = _make_handler()
+
+ result = handler.analyze_initial("test query", "unstructured text")
+
+ mock_llm.invoke.assert_not_called()
+ assert "No sources were found" in result["content"]
+
+ def test_nonempty_results_invoke_llm(self):
+ handler, mock_llm = _make_handler()
+ results = [
+ {"full_content": "content", "link": "http://a.com", "title": "A"},
+ ]
+
+ result = handler.analyze_initial("test query", results)
+
+ mock_llm.invoke.assert_called_once()
+ assert len(result["documents"]) == 1
+
+
+class TestAnalyzeFollowupNoSourcesGuard:
+ def test_empty_results_and_no_knowledge_skip_llm(self):
+ """Neither sources nor previous knowledge -> refuse synthesis."""
+ handler, mock_llm = _make_handler()
+
+ result = handler.analyze_followup("question", [], "", 0)
+
+ mock_llm.invoke.assert_not_called()
+ assert result["documents"] == []
+ assert "No sources were found" in result["content"]
+
+ def test_empty_results_skip_fact_check_too(self):
+ """The fact-check LLM pass is also skipped by the guard."""
+ handler, mock_llm = _make_handler(
+ settings_snapshot={"general.enable_fact_checking": True}
+ )
+
+ handler.analyze_followup("question", [], "", 0)
+
+ mock_llm.invoke.assert_not_called()
+
+ def test_empty_results_with_knowledge_invoke_llm(self):
+ """Previous knowledge alone is a legitimate citation source."""
+ handler, mock_llm = _make_handler()
+
+ result = handler.analyze_followup(
+ "question", [], "Prior findings: X is true [1].", 0
+ )
+
+ mock_llm.invoke.assert_called_once()
+ assert "No sources were found" not in result["content"]
+
+ def test_whitespace_knowledge_counts_as_empty(self):
+ handler, mock_llm = _make_handler()
+
+ result = handler.analyze_followup("question", [], " \n", 0)
+
+ mock_llm.invoke.assert_not_called()
+ assert "No sources were found" in result["content"]
+
+ def test_nonempty_results_invoke_llm(self):
+ handler, mock_llm = _make_handler()
+ results = [
+ {"full_content": "content", "link": "http://a.com", "title": "A"},
+ ]
+
+ result = handler.analyze_followup("question", results, "", 0)
+
+ mock_llm.invoke.assert_called_once()
+ assert len(result["documents"]) == 1
diff --git a/tests/web_search_engines/engines/test_local_search_coverage.py b/tests/web_search_engines/engines/test_local_search_coverage.py
index 2a81c7b10e..3a425f30ee 100644
--- a/tests/web_search_engines/engines/test_local_search_coverage.py
+++ b/tests/web_search_engines/engines/test_local_search_coverage.py
@@ -719,14 +719,16 @@ class TestGetPreviews:
# Top-level search error handling
# ---------------------------------------------------------------------------
class TestSearchTopLevelErrors:
- def test_search_returns_empty_on_exception(self):
+ def test_search_raises_on_exception(self):
+ """A failed search raises instead of masquerading as no results."""
+ import pytest
+
engine = _make_engine(settings_snapshot={"_username": "u"})
with patch(_PATCH_LIB_SERVICE) as lib_svc:
lib_svc.side_effect = Exception("total failure")
- results = engine.search("q")
-
- assert results == []
+ with pytest.raises(Exception, match="total failure"):
+ engine.search("q")
def test_search_empty_when_no_results_across_collections(self):
engine = _make_engine(settings_snapshot={"_username": "u"})
diff --git a/tests/web_search_engines/engines/test_search_engine_collection.py b/tests/web_search_engines/engines/test_search_engine_collection.py
index 6956a41640..adca744597 100644
--- a/tests/web_search_engines/engines/test_search_engine_collection.py
+++ b/tests/web_search_engines/engines/test_search_engine_collection.py
@@ -463,7 +463,9 @@ class TestSearch:
assert results == []
def test_search_exception(self):
- """Search handles exceptions gracefully."""
+ """Search re-raises so failures are not mistaken for no results."""
+ import pytest
+
from local_deep_research.web_search_engines.engines.search_engine_collection import (
CollectionSearchEngine,
)
@@ -500,9 +502,10 @@ class TestSearch:
collection_name="Test Collection",
settings_snapshot=settings,
)
- results = engine.search("test query")
-
- assert results == []
+ with pytest.raises(
+ Exception, match="RAG service error"
+ ):
+ engine.search("test query")
class TestGetDocumentUrl:
diff --git a/tests/web_search_engines/engines/test_search_engine_library.py b/tests/web_search_engines/engines/test_search_engine_library.py
index 4f4c7dea0a..eaf6c8ee9a 100644
--- a/tests/web_search_engines/engines/test_search_engine_library.py
+++ b/tests/web_search_engines/engines/test_search_engine_library.py
@@ -219,7 +219,9 @@ class TestSearch:
assert "/library/document/123" in results[0]["url"]
def test_search_exception(self):
- """Search handles exceptions."""
+ """Search re-raises so failures are not mistaken for no results."""
+ import pytest
+
from local_deep_research.web_search_engines.engines.search_engine_library import (
LibraryRAGSearchEngine,
)
@@ -242,9 +244,8 @@ class TestSearch:
)
engine = LibraryRAGSearchEngine(settings_snapshot=settings)
- results = engine.search("test query")
-
- assert results == []
+ with pytest.raises(Exception, match="Service error"):
+ engine.search("test query")
class TestGetPreviews:
diff --git a/tests/web_search_engines/engines/test_search_engine_library_coverage.py b/tests/web_search_engines/engines/test_search_engine_library_coverage.py
index fa258a6eaf..5e57e6d2bb 100644
--- a/tests/web_search_engines/engines/test_search_engine_library_coverage.py
+++ b/tests/web_search_engines/engines/test_search_engine_library_coverage.py
@@ -49,7 +49,7 @@ class TestLibraryRAGSearchEngineInit:
class TestSearch:
def test_search_no_username_returns_empty(self):
- engine = _make_engine(settings_snapshot={})
+ engine = _make_engine(settings_snapshot={"_username": None})
result = engine.search("test query")
assert result == []
@@ -61,13 +61,16 @@ class TestSearch:
result = engine.search("test query")
assert result == []
- def test_search_exception_returns_empty(self):
+ def test_search_exception_propagates(self):
+ """A failed search raises instead of masquerading as no results."""
+ import pytest
+
engine = _make_engine()
with patch(
f"{MODULE}.LibraryService", side_effect=RuntimeError("fail")
):
- result = engine.search("test query")
- assert result == []
+ with pytest.raises(RuntimeError, match="fail"):
+ engine.search("test query")
def test_search_collection_no_rag_index_skips(self):
engine = _make_engine()
@@ -90,8 +93,10 @@ class TestSearch:
assert result == []
- def test_search_collection_exception_continues(self):
- """Exception in one collection doesn't stop search of others."""
+ def test_all_collections_fail_with_no_results_raises(self):
+ """Zero results + collection errors must not look like no matches."""
+ import pytest
+
engine = _make_engine()
mock_service = MagicMock()
mock_service.get_all_collections.return_value = [
@@ -103,9 +108,61 @@ class TestSearch:
f"{MODULE}.get_user_db_session",
side_effect=RuntimeError("db error"),
):
- result = engine.search("test query")
+ with pytest.raises(RuntimeError, match="failed for 1"):
+ engine.search("test query")
- assert result == []
+ def test_search_collection_exception_continues(self):
+ """Exception in one collection doesn't stop search of others."""
+ engine = _make_engine()
+ mock_service = MagicMock()
+ mock_service.get_all_collections.return_value = [
+ {"id": "col1", "name": "Collection 1"},
+ {"id": "col2", "name": "Collection 2"},
+ ]
+
+ mock_session = MagicMock()
+ mock_session.__enter__ = Mock(return_value=mock_session)
+ mock_session.__exit__ = Mock(return_value=False)
+ mock_rag_index = MagicMock()
+ mock_rag_index.embedding_model = "all-MiniLM-L6-v2"
+ mock_rag_index.embedding_model_type = MagicMock(
+ value="sentence_transformers"
+ )
+ mock_rag_index.chunk_size = 1000
+ mock_rag_index.chunk_overlap = 200
+ mock_session.query.return_value.filter_by.return_value.first.return_value = mock_rag_index
+
+ # col1 fails when its RAG service is created; col2 succeeds
+ mock_doc = MagicMock()
+ mock_doc.metadata = {}
+ mock_doc.page_content = "matching text from collection 2"
+ mock_vector_store = MagicMock()
+ mock_vector_store.similarity_search_with_score.return_value = [
+ (mock_doc, 0.5)
+ ]
+ mock_rag_service = MagicMock()
+ mock_rag_service.__enter__ = Mock(return_value=mock_rag_service)
+ mock_rag_service.__exit__ = Mock(return_value=False)
+ mock_rag_service.get_rag_stats.return_value = {"indexed_documents": 1}
+ mock_rag_service.load_or_create_faiss_index.return_value = (
+ mock_vector_store
+ )
+
+ with patch(f"{MODULE}.LibraryService", return_value=mock_service):
+ with patch(
+ f"{MODULE}.get_user_db_session", return_value=mock_session
+ ):
+ with patch(
+ f"{MODULE}.LibraryRAGService",
+ side_effect=[
+ RuntimeError("col1 broken"),
+ mock_rag_service,
+ ],
+ ):
+ result = engine.search("test query")
+
+ assert len(result) == 1
+ assert result[0]["metadata"]["collection_name"] == "Collection 2"
def test_search_no_results_across_collections(self):
"""When all collections have no indexed docs, returns empty."""