Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(research): dedup citations on (url, snippet) so a source's later excerpts survive (#5894) (#5900)

LearningCircuit1ea34f0c12c2
diff --git a/changelog.d/5894.bugfix.md b/changelog.d/5894.bugfix.md
new file mode 100644
index 0000000000..7b2662411a
--- /dev/null
+++ b/changelog.d/5894.bugfix.md
@@ -0,0 +1,3 @@
+Fixed `SearchResultsCollector` (LangGraph agent strategy) discarding a source's later excerpts. Citations were deduplicated on the canonical URL alone, so a URL reached by several queries kept only the first occurrence's snippet — the agent read three excerpts and the bibliography, `research_resources` and the API `sources` payload retained one. The dedup key is now the pair `(canonical URL, snippet)`: a repeat with the *same* snippet still collapses onto its existing `[N]` (the #5381 behaviour), while a repeat with a genuinely different snippet becomes its own entry with its own `[N]`. Each entry owns exactly one citation index, so `CitationFormatter`'s index → URL map stays 1:1. Snippets are compared after normalising whitespace, case and trailing ellipsis/punctuation, over a `blake2b` digest of the *whole* passage — never a prefix, which cannot tell "one passage truncated twice" from "two passages that open alike". There is deliberately no cap on how many excerpts one source may contribute: folding a distinct excerpt onto an existing citation would show the model text under a number that resolves to different text, and the count is already bounded by the number of queries run.
+
+`all_links_of_system` therefore holds one entry per distinct excerpt rather than one per source, and every "N sources" number now counts distinct canonical URLs via the new `search_utilities.count_distinct_sources` — the agent step heartbeat, `_finalize`'s accumulated-sources milestone and its two citation-skip warnings, and the detailed report's `initial_sources` metadata. `initial_sources` now counts distinct sources rather than raw entries for **every** strategy, not only the LangGraph agent: `report_generator` reads `all_links_of_system` whatever produced it, and `source_based`, `focused_iteration` and `topic_organization` have no cross-call dedup at all, so a run that revisits a URL across `analyze_topic()` calls will report a lower — and now correct — number than before (e.g. 4 raw entries reported as 3 sources). `format_links_to_markdown` still renders one `## Sources` line per source, now listing every citation number that source owns (`[1, 4] Title`), and it groups on the same field the collector keys on (`link` first, via the new `source_url_field`) so a result carrying divergent `url` and `link` cannot split one source across two lines.
diff --git a/src/local_deep_research/advanced_search_system/strategies/langgraph_agent_strategy.py b/src/local_deep_research/advanced_search_system/strategies/langgraph_agent_strategy.py
index 630f44a445..52f60bd967 100644
--- a/src/local_deep_research/advanced_search_system/strategies/langgraph_agent_strategy.py
+++ b/src/local_deep_research/advanced_search_system/strategies/langgraph_agent_strategy.py
@@ -8,10 +8,12 @@ questions can be decomposed into subtopics researched in parallel by subagents.
from __future__ import annotations
+import hashlib
import math
import re
import threading
import time
+import unicodedata
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime
from typing import Any, Dict, Optional
@@ -36,7 +38,10 @@ from ...utilities.chunk_anchor import (
is_library_chunk_result,
is_library_document_link,
)
-from ...utilities.search_utilities import _sanitize_sources_field
+from ...utilities.search_utilities import (
+ _sanitize_sources_field,
+ count_distinct_sources,
+)
from ...utilities.thread_context import get_search_context, search_context
from ...utilities.url_utils import (
CHUNK_DISPLAY_KEY,
@@ -143,6 +148,76 @@ def _citation_dedup_key(link: Any) -> str:
return link
+def _snippet_dedup_key(result: dict) -> str:
+ """Second half of the citation dedup key: the snippet, normalised.
+
+ Reads the snippet with the SAME expression ``_format_results``
+ renders it with, so "the text the model was shown" and "the text the
+ key was computed from" are the same text. (``get("snippet", ...)``
+ and not ``get("snippet") or ...``: an engine that sets ``snippet`` to
+ an empty string has said the excerpt is empty, and both the renderer
+ and this key honour that instead of falling through to ``body``.)
+
+ The normalisation answers "is this the same passage?", not "is this
+ the same string", and every step collapses spellings of one passage
+ rather than distinct passages:
+
+ * whitespace runs (newlines included) collapse to one space and the
+ ends are stripped, so a re-wrapped or trailing-space preview is not
+ a second passage;
+ * Unicode is normalised to NFC, so one engine's precomposed ``café``
+ and another's decomposed ``cafe\u0301`` — identical on screen, and
+ the same passage to any reader — hash alike instead of becoming two
+ citation indices;
+ * case is folded;
+ * a trailing ellipsis or sentence punctuation is dropped, so
+ ``"…text…"`` and ``"…text"`` agree.
+
+ Returns ``""`` — a sentinel, not a hash — for anything with no
+ content: a missing/empty snippet, whitespace or punctuation only, or
+ a non-``str`` (an engine handing back a list, which would also be
+ unhashable in the tuple key). Hashing ``""`` instead would give every
+ content-free snippet one shared REAL key, which is the same
+ collapsing behaviour by accident; the sentinel says it on purpose, so
+ a snippet-less repeat of a known URL collapses exactly as it did
+ before this key existed.
+
+ The digest is taken over the WHOLE normalised passage. An earlier
+ revision of this change compared a bounded prefix, meaning to collapse
+ two truncations of one passage — but a prefix cannot tell that case
+ apart from two DIFFERENT passages that open the same way, and merging
+ those silently discards an excerpt, which is the one outcome this key
+ exists to prevent. Hashing also bounds the key: a multi-kilobyte
+ snippet costs 32 characters in the map instead of being retained in
+ full. Not a security boundary — it only answers "same text?" — but a
+ cryptographic digest is the cheapest way to make an accidental
+ collision unreachable at report volumes.
+
+ Normalising for the KEY only: the stored entry keeps the engine's
+ snippet verbatim, so nothing a reader sees is altered.
+ """
+ snippet = result.get("snippet", result.get("body", ""))
+ if not isinstance(snippet, str):
+ return ""
+ # ``…`` is the single-character ellipsis; the three-dot spelling is
+ # covered by stripping "." itself.
+ # NFC before casefolding: ``str.casefold`` does not compose, so a
+ # decomposed spelling stays decomposed and would digest differently.
+ normalised = (
+ unicodedata.normalize("NFC", " ".join(snippet.split()))
+ .casefold()
+ .rstrip(" .…")
+ )
+ if not normalised:
+ # Empty, whitespace-only or punctuation-only: no passage here.
+ # ONE guard rather than an extra ``not snippet`` fast path above,
+ # so there is no branch a test cannot reach.
+ return ""
+ return hashlib.blake2b(
+ normalised.encode("utf-8"), digest_size=16
+ ).hexdigest()
+
+
def _is_library_citation(value: Any) -> bool:
"""Return ``True`` if *value* addresses a library document.
@@ -254,6 +329,32 @@ class SearchResultsCollector:
linear fallback returns the same answer — but it is why the skip is
described as exact for the append-only contract rather than
unconditionally.
+
+ .. note::
+ Citations are deduplicated on the PAIR ``(canonical url,
+ snippet)``, not on the URL alone. One entry per pair, one
+ citation index per entry — so a URL found again with the SAME
+ snippet still collapses onto its existing ``[N]`` (#5381), while
+ a URL found again with a DIFFERENT snippet gets its own entry and
+ its own ``[N]`` instead of having that snippet discarded (#5894).
+
+ The consequence is that ``_all_links`` holds one entry per
+ OCCURRENCE-OF-DISTINCT-EVIDENCE, not one per source, so
+ ``len()`` of it is not a source count. Use
+ ``count_distinct_sources`` — the same grouping
+ ``format_links_to_markdown`` renders one line per. (The list
+ already had that shape from ``source_based`` /
+ ``focused_iteration`` / ``topic_organization``, which extend it
+ with raw engine dicts and no URL dedup at all.)
+
+ What must NOT be done is to append an entry while reusing another
+ entry's index: that makes index -> entry one-to-many, and
+ ``CitationFormatter.apply_inline_hyperlinks`` builds its
+ index -> url map LAST-WINS, so a later spelling of the URL —
+ credentials and all — silently repoints an earlier citation's
+ hyperlink in the report and in ``research_resources``. An earlier
+ attempt at #5894 did exactly that. One index per entry keeps that
+ map 1:1.
"""
def __init__(self, all_links: list | None = None) -> None:
@@ -273,7 +374,17 @@ class SearchResultsCollector:
self._sources_seen: set[str] = set()
self._lock = threading.Lock()
self._all_links = all_links if all_links is not None else []
+ # Canonical URL -> the FIRST citation index allocated for it.
+ # No longer the dedup key (see ``_pair_to_index``): a source can
+ # own several indices now, and this map answers the different
+ # question ``find_by_url`` asks — "which citation does this URL
+ # resolve to?" — for which the first one is the stable answer.
self._url_to_index: dict[str, str] = {}
+ # ``(canonical URL, snippet key)`` -> citation index. THE dedup
+ # key. Persists across ``reset()`` for the same reason
+ # ``_url_to_index`` does: a source re-cited in a later subsection
+ # must keep its number.
+ self._pair_to_index: dict[tuple[str, str], str] = {}
self._index_to_result: dict[str, dict] = {}
# Highest seeded citation index — used by ``add_results`` and
# ``find_or_add_result`` so newly-allocated indices never collide
@@ -313,6 +424,14 @@ class SearchResultsCollector:
idx_str = str(idx)
if key and key not in self._url_to_index:
self._url_to_index[key] = idx_str
+ if key:
+ # A seed already holds its own snippet, so register
+ # its pair too — otherwise the first ``add_results``
+ # that repeats it would allocate a second entry for
+ # evidence the seeded entry already carries.
+ self._pair_to_index.setdefault(
+ (key, _snippet_dedup_key(r)), idx_str
+ )
if idx_str not in self._index_to_result:
self._index_to_result[idx_str] = r
try:
@@ -340,20 +459,30 @@ class SearchResultsCollector:
and pass ``indexed`` to ``_format_results`` so deduped ``[N]``
markers stay in sync. See ``changelog.d/5381.breaking.md``.
- Returns ``(start_idx, indexed_copies)``:
+ Returns ``(start_idx, indexed)``:
* ``start_idx`` — 0-based offset of the first added result, measured
against ``_all_links``. Used for diagnostic logging; do **not**
slice ``self._results[start_idx:]`` to recover "the just-added"
results, because duplicates are appended to ``_results`` but not
to ``_all_links``, so the two lists grow at different rates.
- * ``indexed_copies`` — the indexed dicts that were appended to
+ * ``indexed`` — the indexed dicts that were appended to
``_results`` in this batch (each carrying the assigned ``index``).
Callers formatting the batch for the LLM should pass this list
to ``_format_results`` so the displayed ``[N]`` markers match the
collector's stored citation indices — including for duplicates
that reuse an existing index instead of getting a new one.
+ READ-ONLY. The list is new but its dicts are the LIVE entries —
+ the very objects held by ``_results``, ``_index_to_result`` and
+ ``all_links_of_system``. They are copies of the ENGINE's dicts
+ (so ingest does not mutate engine output), not copies of
+ collector state. Mutating one rewrites what the bibliography
+ renders and, if ``link`` or ``index`` is touched, desyncs the
+ dedup maps that were keyed from it. No copy is taken because
+ nothing in-tree writes to them and the results carry whole page
+ bodies; a caller that needs to mutate must deep-copy first.
+
The entire operation runs under a single lock acquisition so that
citation indices are never duplicated.
"""
@@ -443,10 +572,22 @@ class SearchResultsCollector:
# onto one line, so keying on the raw spelling would report
# three sources for a report that renders one.
key = _citation_dedup_key(link)
- if key and key in self._url_to_index:
- # Duplicate URL — reuse the existing citation index
- # so repeated search hits collapse to a single [N].
- r["index"] = self._url_to_index[key]
+ # The dedup key is the PAIR. Keying on the URL alone
+ # collapsed every later hit of a source onto the first
+ # one's entry and DISCARDED its snippet, so a URL reached
+ # by three queries contributed three excerpts the model
+ # read and one that survived into the bibliography
+ # (#5894). Keying on the pair keeps the genuine-duplicate
+ # collapse #5381 was written for — same URL, same snippet
+ # — and gives a genuinely different excerpt its own entry
+ # instead.
+ snippet_key = _snippet_dedup_key(r)
+ reuse_idx = self._reuse_index(key, snippet_key)
+ if reuse_idx is not None:
+ # Same URL AND same snippet — reuse the existing
+ # citation index so repeated search hits collapse to
+ # a single [N].
+ r["index"] = reuse_idx
self._prefer_anchored_link(r["index"], link)
logger.debug(
"add_results: dedup reuse url={} -> [{}]",
@@ -462,7 +603,11 @@ class SearchResultsCollector:
new_idx = str(self._max_idx)
r["index"] = new_idx
if key:
- self._url_to_index[key] = new_idx
+ self._pair_to_index[(key, snippet_key)] = new_idx
+ # ``setdefault``: a source's SECOND excerpt must
+ # not repoint ``find_by_url`` (and the fetch
+ # tool's "[N] for this url") at the newer entry.
+ self._url_to_index.setdefault(key, new_idx)
# Track by index either way so find_by_index resolves
# linkless entries too.
self._index_to_result[new_idx] = r
@@ -499,6 +644,44 @@ class SearchResultsCollector:
indexed.append(r)
return start_idx, indexed
+ def _reuse_index(self, key: str, snippet_key: str) -> str | None:
+ """The citation index this occurrence collapses onto, or ``None``.
+
+ Same URL AND same snippet reuses the existing entry — the
+ genuine-duplicate collapse #5381 exists for. A different snippet
+ under a known URL returns ``None``, so the caller allocates an
+ entry of its own for it instead of discarding the text (#5894).
+
+ A content-free snippet is the exception: there is no evidence to
+ preserve, so it collapses on the URL alone, exactly as every
+ occurrence did before the snippet joined the key. Without this a
+ snippet-less re-fetch of a cited page would allocate a second
+ citation for nothing.
+
+ There is deliberately NO cap on how many excerpts one source may
+ contribute. A distinct excerpt is distinct evidence, and the only
+ way to fold it onto an existing citation is to show the model
+ text under a number that resolves to different text — a citation
+ pointing at evidence nobody read. The count is already bounded by
+ the number of queries run (``iterations`` x
+ ``questions_per_iteration``), so unbounded growth is not
+ reachable through the normal loop. If a backstop is ever wanted
+ it must SUPPRESS the extra entry, not alias it onto a citation
+ that means something else.
+
+ ONE helper because both ingest paths must ask this identically.
+ The first attempt at #5894 changed ``add_results`` alone and left
+ the fetch path collapsing on the URL, so a fetched excerpt was
+ still dropped.
+
+ Callers must hold ``_lock``.
+ """
+ if not key:
+ return None
+ if not snippet_key:
+ return self._url_to_index.get(key)
+ return self._pair_to_index.get((key, snippet_key))
+
def _prefer_anchored_link(self, index: str, link: str) -> None:
"""Record a chunk-anchored spelling ALONGSIDE the stored citation.
@@ -557,10 +740,19 @@ class SearchResultsCollector:
lookup and append under one lock prevents two fetches of the same URL
from allocating separate citation indices.
- Dedup uses the same canonical key as ``add_results`` and as the
- rendered bibliography, so fetching a document the agent already
- saw in a search hit reuses that citation index no matter which of
- the document's URL spellings the agent typed back.
+ Dedup uses the same ``(canonical url, snippet)`` pair as
+ ``add_results``, so fetching a document the agent already saw
+ reuses that citation index no matter which of the document's URL
+ spellings the agent typed back — and, when the fetched text is
+ genuinely different evidence from the search snippet, gets its own
+ entry and its own index rather than having that text discarded.
+
+ Both ingest paths must ask the identical question. The first
+ attempt at #5894 changed ``add_results`` only, so a fetch of an
+ already-cited URL still collapsed onto the search hit's entry and
+ dropped the fetched excerpt — the same one-sided omission that
+ ``_prefer_anchored_link`` (wired into both paths from the start)
+ exists to avoid.
"""
# Strip BEFORE the key is computed and before ``_sources`` records
# the URL. Doing it at the end of the new-entry branch (as the first
@@ -578,6 +770,7 @@ class SearchResultsCollector:
# Computed outside the lock: pure, and ``canonical_url_key`` is
# LRU-cached.
key = _citation_dedup_key(url)
+ snippet_key = _snippet_dedup_key(result)
with self._lock:
# Record the source before branching: all three paths below end
# with *url* tracked (reused via either dedup path, or freshly
@@ -614,9 +807,10 @@ class SearchResultsCollector:
# preserves the padded form. The int is only for the return
# value and ordering.
reused_key: str | None = None
- if key and key in self._url_to_index:
+ existing_idx = self._reuse_index(key, snippet_key)
+ if existing_idx is not None:
try:
- reused_key = str(self._url_to_index[key])
+ reused_key = str(existing_idx)
reused = int(reused_key)
except (ValueError, TypeError):
reused = None
@@ -649,6 +843,18 @@ class SearchResultsCollector:
tracked_url = tracked.get("link", tracked.get("url", ""))
if _citation_dedup_key(tracked_url) != key:
continue
+ # Same snippet too, not just the same URL — an
+ # outside entry holding a DIFFERENT excerpt of this
+ # source is not this occurrence, and reusing its
+ # index would discard the fetched text exactly as the
+ # URL-only key did. A content-free snippet still
+ # matches on the URL alone, mirroring
+ # ``_reuse_index``.
+ if (
+ snippet_key
+ and _snippet_dedup_key(tracked) != snippet_key
+ ):
+ continue
idx = tracked.get("index")
if idx is None:
continue
@@ -657,7 +863,10 @@ class SearchResultsCollector:
reused = int(idx_str)
except (ValueError, TypeError):
continue
- self._url_to_index[key] = idx_str
+ self._url_to_index.setdefault(key, idx_str)
+ self._pair_to_index.setdefault(
+ (key, _snippet_dedup_key(tracked)), idx_str
+ )
# Externally appended, so it never passed the ingest
# strip. Unconditional: the entry is already in
# ``_all_links`` and therefore already reader-visible,
@@ -720,7 +929,11 @@ class SearchResultsCollector:
tracked["link"] = tracked["url"]
self._results.append(tracked)
if key:
- self._url_to_index[key] = str(index)
+ self._pair_to_index[(key, snippet_key)] = str(index)
+ # ``setdefault`` for the same reason as in
+ # ``add_results``: the first index a URL got is the one
+ # ``find_by_url`` keeps answering with.
+ self._url_to_index.setdefault(key, str(index))
# ``_sources`` was already recorded above, before the
# dedup branches.
self._index_to_result[str(index)] = tracked
@@ -785,10 +998,15 @@ class SearchResultsCollector:
O(1) via ``_index_to_result`` with a linear-scan fallback for
externally-mutated entries.
- Returns a shallow COPY, matching ``results`` and ``sources``: the
- stored dict is the very object ``_url_to_index`` was keyed from, so
- a caller that rewrote its ``link`` in place would silently desync
- the dedup map from ``_all_links``.
+ READ-ONLY, and only shallowly isolated. The returned dict is a
+ new top-level ``dict``, matching ``results`` and ``sources`` — so
+ rebinding a top-level key such as ``link`` cannot desync the dedup
+ maps from ``_all_links``, which is what the copy is for. Its
+ VALUES are not copied: a nested ``metadata`` dict is the live
+ object the collector holds, so mutating it mutates collector
+ state. Deep-copying is deliberately not done — entries carry whole
+ page bodies and nothing in-tree writes to them — so a caller that
+ needs to mutate must copy first.
"""
if not isinstance(idx, int) or idx < 1:
return None
@@ -820,6 +1038,16 @@ class SearchResultsCollector:
@property
def results(self) -> list[dict]:
+ """Every indexed result recorded since the last ``reset()``.
+
+ READ-ONLY. The list is new — appending to it does not touch the
+ collector — but its dicts are the LIVE entries, shared with
+ ``_index_to_result`` and ``all_links_of_system``; only the outer
+ list is copied. Mutating an entry rewrites what the bibliography
+ renders, and rewriting its ``link`` or ``index`` desyncs the dedup
+ maps that were keyed from it. Same contract as ``find_by_index``,
+ one level shallower: that one at least rebinds the outer dict.
+ """
with self._lock:
return list(self._results)
@@ -2020,7 +2248,11 @@ class LangGraphAgentStrategy(BaseSearchStrategy):
Extracted from the ``analyze_topic`` stream loop so it can be
unit-tested without driving the full LangGraph stream.
"""
- sources_so_far = len(self.all_links_of_system)
+ # SOURCES, not entries: ``all_links_of_system`` holds one entry
+ # per distinct (url, snippet) pair, so its length counts pieces
+ # of evidence. The user reads this line as "how many sources has
+ # it found", which is what the ## Sources block will show.
+ sources_so_far = count_distinct_sources(self.all_links_of_system)
names = getattr(self, "_tool_names", []) or []
if sources_so_far == 0:
return (
@@ -2649,6 +2881,10 @@ class LangGraphAgentStrategy(BaseSearchStrategy):
# guarantees, and the empty-collector condition is also reachable
# from chat follow-ups. Until that is redesigned, make the skip
# loud and report whether the raw answer already contains markers.
+ # Counted, not measured — see ``count_distinct_sources``. Every
+ # "N sources" number below is what the ## Sources block renders,
+ # not how many entries back it.
+ accumulated_sources = count_distinct_sources(self.all_links_of_system)
if (
not all_search_results
and self.all_links_of_system
@@ -2660,7 +2896,7 @@ class LangGraphAgentStrategy(BaseSearchStrategy):
if existing_markers:
logger.warning(
f"Citation pass skipped: no new sources collected in "
- f"this call although {len(self.all_links_of_system)} "
+ f"this call although {accumulated_sources} "
f"are accumulated; the raw answer already contains "
f"{len(existing_markers)} inline [N]/【N】 marker(s) and "
f"will be preserved as-is (query '{query[:80]}')"
@@ -2668,7 +2904,7 @@ class LangGraphAgentStrategy(BaseSearchStrategy):
else:
logger.warning(
f"Citation pass skipped: no new sources collected in "
- f"this call although {len(self.all_links_of_system)} "
+ f"this call although {accumulated_sources} "
f"are accumulated, and the raw answer contains no "
f"inline [N]/【N】 markers (query '{query[:80]}')"
)
@@ -2677,13 +2913,13 @@ class LangGraphAgentStrategy(BaseSearchStrategy):
if final_answer != NO_RESULTS_MESSAGE:
if not all_search_results and self.all_links_of_system:
self._update_progress(
- f"Skipping citation synthesis (reusing {len(self.all_links_of_system)} accumulated sources)",
+ f"Skipping citation synthesis (reusing {accumulated_sources} accumulated sources)",
90,
{
"phase": "synthesis",
"type": "milestone",
"new_sources": 0,
- "accumulated_sources": len(self.all_links_of_system),
+ "accumulated_sources": accumulated_sources,
"citation_pass_skipped": True,
},
)
diff --git a/src/local_deep_research/report_generator.py b/src/local_deep_research/report_generator.py
index f250fb345c..cfc0f80860 100644
--- a/src/local_deep_research/report_generator.py
+++ b/src/local_deep_research/report_generator.py
@@ -1296,6 +1296,11 @@ class IntegratedReportGenerator:
# Format links from search system
# Get utilities module dynamically to avoid circular imports
utilities = importlib.import_module("local_deep_research.utilities")
+ # Imported here for the same reason, and directly rather than off
+ # the module object above so the count is computed by the real
+ # grouping helper.
+ from .utilities.search_utilities import count_distinct_sources
+
formatted_all_links = (
utilities.search_utilities.format_links_to_markdown(
all_links=self.search_system.all_links_of_system
@@ -1338,7 +1343,15 @@ class IntegratedReportGenerator:
# Create metadata dictionary
metadata = {
"generated_at": datetime.now(UTC).isoformat(),
- "initial_sources": len(self.search_system.all_links_of_system),
+ # SOURCES, not entries. ``all_links_of_system`` holds one
+ # entry per distinct (url, snippet) pair under the LangGraph
+ # strategy, and raw un-deduped engine dicts under the others,
+ # so its length counts occurrences. This number is reported
+ # as "sources", so it must group the way the ## Sources block
+ # above does.
+ "initial_sources": count_distinct_sources(
+ self.search_system.all_links_of_system
+ ),
"sections_researched": len(structure),
"searches_per_section": self.searches_per_section,
"query": query,
diff --git a/src/local_deep_research/utilities/search_utilities.py b/src/local_deep_research/utilities/search_utilities.py
index c7162a664e..35570320dd 100644
--- a/src/local_deep_research/utilities/search_utilities.py
+++ b/src/local_deep_research/utilities/search_utilities.py
@@ -277,6 +277,58 @@ def _owned_chunk_display(recorded: object, canon: str) -> str | None:
return display if canonical_url_key(display) == canon else None
+def source_url_field(link: Dict) -> object:
+ """The field a source's identity is read from: ``link`` first.
+
+ ``SearchResultsCollector`` keys every citation on
+ ``_citation_dedup_key(result["link"])``, and
+ :func:`_citation_dedup_key`'s docstring requires that key to be the
+ one this module groups the rendered bibliography by. While the
+ collector kept at most one entry per source the disagreement was
+ invisible — with a single entry there is nothing to group. Now that a
+ source can own several entries (one per distinct snippet), a result
+ carrying BOTH fields with different values would land in a render
+ group of its own and give one source two ``## Sources`` lines under
+ different numbers.
+
+ ``link`` wins because it is what the collector keys on, what
+ ``_format_results`` shows the agent next to the ``[N]`` marker, and
+ what ``find_by_url`` resolves — i.e. it is what the citation index
+ actually means. ``url`` remains the fallback for the raw engine dicts
+ the non-LangGraph strategies extend ``all_links_of_system`` with,
+ some of which set only that key.
+ """
+ return link.get("link") or link.get("url") or ""
+
+
+def count_distinct_sources(all_links: List[Dict]) -> int:
+ """How many sources :func:`format_links_to_markdown` will render.
+
+ ``len(all_links_of_system)`` counts OCCURRENCES, not sources: the
+ LangGraph collector stores one entry per ``(url, snippet)`` pair, and
+ ``source_based`` / ``focused_iteration`` / ``topic_organization``
+ extend the list with raw engine dicts and no URL dedup at all. Both
+ shapes put one source in the list several times, so every "N sources"
+ number must group the way the bibliography does rather than take a
+ length.
+
+ Grouped with the same expression and the same canonical key the
+ renderer uses, so the count and the number of rendered lines cannot
+ drift.
+ """
+ seen: set[str] = set()
+ for link in all_links or []:
+ if not isinstance(link, dict):
+ continue
+ raw = source_url_field(link)
+ if not isinstance(raw, str):
+ continue
+ canon = canonical_url_key(raw)
+ if canon:
+ seen.add(canon)
+ return len(seen)
+
+
def format_links_to_markdown(all_links: List[Dict]) -> str:
parts: list[str] = []
logger.info(f"Formatting {len(all_links)} links to markdown...")
@@ -304,7 +356,7 @@ def format_links_to_markdown(all_links: List[Dict]) -> str:
# utm_*/fbclid clutter and credentials stay out of the report.
canon_to_display: dict[str, str] = {}
for link in all_links:
- raw = link.get("url") or link.get("link") or ""
+ raw = source_url_field(link)
# Skipped, not coerced. These dicts reach here straight from
# engine output on the non-LangGraph strategies, and
# canonical_url_key raises on a non-str. Stringifying instead
@@ -373,7 +425,7 @@ def format_links_to_markdown(all_links: List[Dict]) -> str:
# Emit each unique source once, in first-seen order.
seen: set[str] = set()
for link in all_links:
- raw = link.get("url") or link.get("link") or ""
+ raw = source_url_field(link)
# Skipped, not coerced. These dicts reach here straight from
# engine output on the non-LangGraph strategies, and
# canonical_url_key raises on a non-str. Stringifying instead
diff --git a/src/local_deep_research/web/services/report_assembly_service.py b/src/local_deep_research/web/services/report_assembly_service.py
index 26d098fa51..9ac422cbac 100644
--- a/src/local_deep_research/web/services/report_assembly_service.py
+++ b/src/local_deep_research/web/services/report_assembly_service.py
@@ -21,6 +21,7 @@ from sqlalchemy.orm import Session
from ...database.models.research import ResearchHistory, ResearchResource
from ...utilities.search_utilities import format_links_to_markdown
+from ...utilities.url_utils import canonical_url_key
# Line-anchored regexes for the legacy-row guard. See `assemble_full_report`
# for why a substring `in body` check is too loose.
@@ -164,41 +165,84 @@ def _build_sources_markdown(
return format_links_to_markdown(all_links)
+def _format_source_link(row: ResearchResource) -> Optional[Dict[str, str]]:
+ """Map one ``research_resources`` row to the news feed's link shape.
+
+ Shared by :func:`get_research_source_links` and its batched variant so
+ the two cannot drift. Returns ``None`` for a row the feed cannot link
+ to (a non-``http`` URL). Titles fall back to the bare domain when
+ missing and are truncated to 50 chars, matching the existing
+ list-card rendering in the news UI.
+ """
+ url = (row.url or "").strip()
+ if not url.startswith("http"):
+ return None
+ title = (row.title or "").strip()
+ if not title:
+ domain = url.split("//")[-1].split("/")[0]
+ title = domain.replace("www.", "")
+ if len(title) > 50:
+ title = title[:50] + "..."
+ return {"url": url, "title": title}
+
+
+def _dedup_key(url: str) -> str:
+ """Canonical grouping key for a source link.
+
+ The SAME key ``format_links_to_markdown`` groups the bibliography by
+ and ``count_distinct_sources`` counts with, so a "top 3 sources" card,
+ the rendered ``## Sources`` block and the reported source count cannot
+ disagree about what counts as one source. Falls back to the raw URL if
+ canonicalization yields nothing, so an unrecognizable URL stays its
+ own source rather than merging with every other unrecognizable one.
+ """
+ return canonical_url_key(url) or url
+
+
def get_research_source_links(
research_id: str, db_session: Session, limit: int = 3
) -> List[Dict[str, str]]:
- """Top-N source links for a research, in row-insertion order.
+ """Top-N DISTINCT source links for a research, in row-insertion order.
Returns dicts shaped ``{"url": str, "title": str}`` matching the
news feed's ``links`` contract (``news/api.py`` consumers). Titles
are domain-fallback when missing, truncated to 50 chars to match
the existing list-card rendering in the news UI.
+ Deduplicated on :func:`_dedup_key`, first occurrence winning, so
+ ``limit=N`` yields N *distinct* sources rather than N rows. One row
+ per source is not something the caller can assume: a search strategy
+ stores one ``research_resources`` row per piece of evidence, so the
+ same URL legitimately appears several times with different snippets
+ (see #5894), and without this a "top 3 sources" card could render one
+ URL three times. Because dedup happens in Python the row cap cannot
+ be pushed into SQL, so the query is walked lazily and abandoned as
+ soon as ``limit`` distinct sources are in hand.
+
Args:
research_id: The ResearchHistory id.
db_session: Active SQLAlchemy session bound to the user DB.
- limit: Maximum number of links to return.
+ limit: Maximum number of DISTINCT links to return.
"""
rows = (
db_session.query(ResearchResource)
.filter_by(research_id=research_id)
.filter(ResearchResource.url.isnot(None))
.order_by(ResearchResource.id.asc())
- .limit(limit)
- .all()
)
out: List[Dict[str, str]] = []
+ seen: set[str] = set()
for r in rows:
- url = (r.url or "").strip()
- if not url.startswith("http"):
+ if len(out) >= limit:
+ break
+ link = _format_source_link(r)
+ if link is None:
continue
- title = (r.title or "").strip()
- if not title:
- domain = url.split("//")[-1].split("/")[0]
- title = domain.replace("www.", "")
- if len(title) > 50:
- title = title[:50] + "..."
- out.append({"url": url, "title": title})
+ key = _dedup_key(link["url"])
+ if key in seen:
+ continue
+ seen.add(key)
+ out.append(link)
return out
@@ -210,8 +254,9 @@ def get_research_source_links_batch(
For news-feed list views that would otherwise fire one query per
research item (N+1). One ``WHERE research_id IN (...)`` query plus
Python-side grouping. Returned dict maps each research_id to its
- top-N links (same shape as :func:`get_research_source_links`).
- Research ids with zero rows map to ``[]``.
+ top-N links (same shape as :func:`get_research_source_links`, and
+ deduplicated the same way — ``limit`` counts DISTINCT sources, not
+ rows). Research ids with zero rows map to ``[]``.
``limit=None`` returns every link for each research (no cap) — used by
the report API, which exposes the full source list rather than a top-N.
@@ -227,18 +272,18 @@ def get_research_source_links_batch(
.order_by(ResearchResource.research_id, ResearchResource.id.asc())
.all()
)
+ seen_by_research: Dict[str, set[str]] = {}
for r in rows:
bucket = result.setdefault(r.research_id, [])
if limit is not None and len(bucket) >= limit:
continue
- url = (r.url or "").strip()
- if not url.startswith("http"):
+ link = _format_source_link(r)
+ if link is None:
+ continue
+ seen = seen_by_research.setdefault(r.research_id, set())
+ key = _dedup_key(link["url"])
+ if key in seen:
continue
- title = (r.title or "").strip()
- if not title:
- domain = url.split("//")[-1].split("/")[0]
- title = domain.replace("www.", "")
- if len(title) > 50:
- title = title[:50] + "..."
- bucket.append({"url": url, "title": title})
+ seen.add(key)
+ bucket.append(link)
return result
diff --git a/tests/advanced_search_system/tools/test_fetch_modes.py b/tests/advanced_search_system/tools/test_fetch_modes.py
index 2a057aace8..7cd4dfb634 100644
--- a/tests/advanced_search_system/tools/test_fetch_modes.py
+++ b/tests/advanced_search_system/tools/test_fetch_modes.py
@@ -14,6 +14,9 @@ import pytest
from local_deep_research.advanced_search_system.strategies.langgraph_agent_strategy import (
SearchResultsCollector,
)
+from local_deep_research.utilities.search_utilities import (
+ count_distinct_sources,
+)
from local_deep_research.advanced_search_system.tools.fetch import (
FETCH_MODES,
build_fetch_tool,
@@ -1032,14 +1035,21 @@ def test_register_in_collector_assigns_new_index():
def test_register_in_collector_fast_path_reuses_without_reappend():
- """Already-tracked URL returns existing index and does not grow _results."""
+ """Already-tracked URL and SAME text returns the existing index and does
+ not grow ``_results``.
+
+ The search hit below carries the excerpt the fetch derives from the
+ page body, so the fetch is a genuine duplicate under the
+ ``(url, snippet)`` dedup key introduced for #5894. The companion test
+ covers the other half — a fetch whose text is new.
+ """
from local_deep_research.advanced_search_system.tools.fetch import (
_register_in_collector,
)
collector = SearchResultsCollector([])
collector.add_results(
- [{"title": "A", "link": "http://a.com", "snippet": "a"}],
+ [{"title": "A", "link": "http://a.com", "snippet": "fetched body"}],
engine_name="web",
)
before = len(collector.results)
@@ -1048,6 +1058,46 @@ def test_register_in_collector_fast_path_reuses_without_reappend():
assert len(collector.results) == before
+def test_register_in_collector_keeps_a_fetched_excerpt_that_is_new():
+ """Fetching a cited page whose text differs from the search snippet
+ must NOT discard that text.
+
+ Driven through the real ``_register_in_collector`` — the helper the
+ ``fetch_content`` tool actually calls — because the first attempt at
+ #5894 changed ``add_results`` only and left this path collapsing on
+ the URL alone, so the fetched excerpt was still lost. No test in that
+ attempt exercised the combination.
+
+ The paired assertion is the point: the same-text case above must keep
+ reusing, or this pair would be satisfied by a collector that never
+ dedupes at all.
+ """
+ from local_deep_research.advanced_search_system.tools.fetch import (
+ _register_in_collector,
+ )
+
+ all_links = []
+ collector = SearchResultsCollector(all_links)
+ collector.add_results(
+ [{"title": "A", "link": "http://a.com", "snippet": "search snippet"}],
+ engine_name="web",
+ )
+
+ idx = _register_in_collector(
+ collector, "http://a.com", "A", "an entirely different page body"
+ )
+
+ assert idx == 2, "the fetched excerpt was collapsed onto the search hit"
+ assert len(all_links) == 2
+ snippets = [entry["snippet"] for entry in all_links]
+ assert "search snippet" in snippets
+ assert "an entirely different page body" in snippets
+ # One source, two excerpts: the URL still resolves to its first
+ # citation and the source is counted once.
+ assert collector.find_by_url("http://a.com") == 1
+ assert count_distinct_sources(all_links) == 1
+
+
class _ListReturningCustomCollector:
"""A custom collector returning a 2-LIST rather than a tuple.
diff --git a/tests/core/test_report_generator.py b/tests/core/test_report_generator.py
index 67f215c65e..9026cd134d 100644
--- a/tests/core/test_report_generator.py
+++ b/tests/core/test_report_generator.py
@@ -379,9 +379,12 @@ class TestFormatFinalReport:
mock_search_system = Mock()
mock_llm = Mock()
mock_search_system.model = mock_llm
+ # Three entries, two sources: ``initial_sources`` counts distinct
+ # sources (the ## Sources grouping), not list length.
mock_search_system.all_links_of_system = [
- {"link": "url1"},
- {"link": "url2"},
+ {"link": "https://one.test/a", "snippet": "first"},
+ {"link": "https://one.test/a", "snippet": "second"},
+ {"link": "https://two.test/b"},
]
from local_deep_research.report_generator import (
diff --git a/tests/strategies/test_langgraph_agent_strategy.py b/tests/strategies/test_langgraph_agent_strategy.py
index df15d1c625..eef2a068e0 100644
--- a/tests/strategies/test_langgraph_agent_strategy.py
+++ b/tests/strategies/test_langgraph_agent_strategy.py
@@ -94,8 +94,15 @@ class TestSearchResultsCollector:
assert indexed[0]["index"] == "2"
def test_add_results_dedupes_within_single_batch(self):
- """Two results with the same URL in one ``add_results`` call reuse
- the first slot's index — the dedup must not only span batches."""
+ """Two results with the same URL AND the same snippet in one
+ ``add_results`` call reuse the first slot's index — the dedup must
+ not only span batches.
+
+ The third result repeats the URL with a DIFFERENT snippet. Since
+ #5894 that is not a duplicate: it gets its own entry and its own
+ citation, so this test cannot be satisfied by a collector that
+ simply never dedupes.
+ """
collector, all_links = self._make_collector()
start, indexed = collector.add_results(
[
@@ -103,6 +110,11 @@ class TestSearchResultsCollector:
{
"title": "First dup",
"link": "http://a.com",
+ "snippet": "a",
+ },
+ {
+ "title": "First, other passage",
+ "link": "http://a.com",
"snippet": "a2",
},
{"title": "Other", "link": "http://b.com", "snippet": "b"},
@@ -111,12 +123,14 @@ class TestSearchResultsCollector:
)
assert start == 0
- assert len(all_links) == 2
- assert len(collector.results) == 3
+ assert len(all_links) == 3
+ assert len(collector.results) == 4
assert indexed[0]["index"] == "1"
assert indexed[1]["index"] == "1" # dedup of indexed[0]
- assert indexed[2]["index"] == "2"
- # First URL registered once in sources, second URL also once.
+ assert indexed[2]["index"] == "2" # different excerpt, own citation
+ assert indexed[3]["index"] == "3"
+ # First URL registered once in sources, second URL also once —
+ # a second excerpt is not a second source.
assert collector.sources.count("http://a.com") == 1
def test_add_results_returns_indexed_copies_with_link_normalized(self):
@@ -144,7 +158,16 @@ class TestSearchResultsCollector:
assert indexed[0]["link"] == "http://a.com"
assert indexed[1]["link"] == "/library/document/doc1/chunks#chunk-0"
- def test_add_results_deduplicates_by_url(self):
+ def test_add_results_deduplicates_on_url_and_snippet(self):
+ """The dedup key is the ``(url, snippet)`` PAIR.
+
+ Same URL and same snippet collapses onto the existing citation —
+ the genuine-duplicate case #5381 was written for. Same URL with a
+ different snippet is different evidence and gets an entry and a
+ citation of its own instead of being discarded (#5894). Both
+ halves are asserted, so neither a URL-only key nor no key at all
+ satisfies this test.
+ """
collector, all_links = self._make_collector()
collector.add_results(
[{"title": "A", "link": "http://a.com", "snippet": "a"}],
@@ -155,6 +178,9 @@ class TestSearchResultsCollector:
collector.add_results(
[
+ # Same URL, same snippet: collapses onto [1].
+ {"title": "A", "link": "http://a.com", "snippet": "a"},
+ # Same URL, different snippet: its own entry and [N].
{
"title": "A Duplicate",
"link": "http://a.com",
@@ -164,9 +190,10 @@ class TestSearchResultsCollector:
],
engine_name="test",
)
- assert len(all_links) == 2
+ assert len(all_links) == 3
assert collector.results[1]["index"] == "1"
assert collector.results[2]["index"] == "2"
+ assert collector.results[3]["index"] == "3"
def test_add_results_formats_library_chunk_url(self):
collector, all_links = self._make_collector()
@@ -302,7 +329,15 @@ class TestSearchResultsCollector:
disagreed about which index a source carries.
"""
all_links = [
- {"title": "Padded", "link": "http://pad.com", "index": "007"},
+ # The seed carries the snippet the fetch below repeats: since
+ # #5894 the dedup key is the (url, snippet) pair, so a seed
+ # with no snippet would not be the same occurrence.
+ {
+ "title": "Padded",
+ "link": "http://pad.com",
+ "index": "007",
+ "snippet": "s",
+ },
]
collector, _ = self._make_collector(all_links)
@@ -331,7 +366,14 @@ class TestSearchResultsCollector:
seeded index, not ``len(_all_links) + 1``, so sparse seeded
indices can't be aliased or overwritten."""
all_links = [
- {"title": "Seed", "link": "http://seed.com", "index": "7"},
+ # Same snippet as the result below — the dedup key is the
+ # (url, snippet) pair since #5894.
+ {
+ "title": "Seed",
+ "link": "http://seed.com",
+ "index": "7",
+ "snippet": "s",
+ },
# Legacy entry with no index — must NOT seed "None".
{"title": "Legacy", "link": "http://legacy.com"},
]
@@ -402,9 +444,11 @@ class TestSearchResultsCollector:
]
)
- # The repeat hit still reuses citation [1] ...
+ # ``find_by_url`` still resolves a.com to its FIRST citation,
+ # even though its two further excerpts got [2] and [3] of their
+ # own — which is why b.com is [4].
assert collector.find_by_url("http://a.com") == 1
- assert collector.find_by_url("http://b.com") == 2
+ assert collector.find_by_url("http://b.com") == 4
# ... but must not vanish from this section's sources. Compared as
# a list so a dropped dedup guard (unconditional append) shows up as
# a repeated entry rather than being flattened away by set().
@@ -1188,13 +1232,13 @@ class TestSearchResultsCollector:
assert collector._url_to_index.get("http://fetch.com") == "1"
assert collector._index_to_result.get("1")["link"] == "http://fetch.com"
- # Subsequent search returns the same URL
+ # Subsequent search returns the same URL with the same excerpt.
start, indexed = collector.add_results(
[
{
"title": "Search second",
"link": "http://fetch.com",
- "snippet": "s",
+ "snippet": "f",
}
],
engine_name="web",
@@ -1204,6 +1248,24 @@ class TestSearchResultsCollector:
assert collector.find_by_url("http://fetch.com") == 1
assert collector.find_by_index(1)["title"] == "Fetch first"
+ # Control: a genuinely different excerpt of the same URL is not
+ # the same occurrence and must not be collapsed away. Without it
+ # this test would pass just as well against a URL-only key.
+ _, indexed2 = collector.add_results(
+ [
+ {
+ "title": "Search third",
+ "link": "http://fetch.com",
+ "snippet": "s",
+ }
+ ],
+ engine_name="web",
+ )
+ assert len(all_links) == 2
+ assert indexed2[0]["index"] == "2"
+ # ...and the first citation still resolves to the fetch entry.
+ assert collector.find_by_url("http://fetch.com") == 1
+
def test_add_results_rejects_malicious_doc_id_path_traversal(self):
"""Malicious doc_id containing path traversal (e.g. ../../etc/passwd)
is rejected — and the original link is left unchanged rather than
@@ -1565,7 +1627,9 @@ class TestFormatResults:
{
"title": "A again",
"link": "http://a.com",
- "snippet": "a2",
+ # Same excerpt, so this really is a duplicate under
+ # the (url, snippet) key and collapses onto [1].
+ "snippet": "a",
},
{"title": "B", "link": "http://b.com", "snippet": "b"},
],
@@ -2895,7 +2959,12 @@ class TestHeartbeatMessage:
)
def test_lists_all_tools_without_more_suffix(self):
- strategy = self._make_strategy(links=[{"link": "http://a.com"}] * 5)
+ # Five DISTINCT sources: the heartbeat counts sources, not
+ # entries, and ``all_links_of_system`` holds one entry per
+ # distinct (url, snippet) pair since #5894.
+ strategy = self._make_strategy(
+ links=[{"link": f"http://a{n}.com"} for n in range(5)]
+ )
strategy._tool_names = [
"web_search",
"search_arxiv",
@@ -3262,7 +3331,13 @@ class TestFetchContentCollectorRegistration:
{
"title": "From search",
"link": "http://example.com/page",
- "snippet": "snip",
+ # The excerpt the fetch below derives from the page
+ # body, so the fetch is a genuine duplicate under the
+ # (url, snippet) key. The other half — a fetch whose
+ # text is NEW — is pinned by
+ # ``test_register_in_collector_keeps_a_fetched_excerpt_that_is_new``
+ # in ``tests/advanced_search_system/tools/test_fetch_modes.py``.
+ "snippet": "full body",
}
],
engine_name="web",
@@ -4608,6 +4683,31 @@ class TestFinalizeCitationLogging:
assert meta.get("citation_pass_skipped") is True
assert meta.get("accumulated_sources") == 1
+ def test_milestone_counts_sources_not_entries(self):
+ """``accumulated_sources`` is read as a source count, and
+ ``all_links_of_system`` holds one entry per distinct
+ ``(url, snippet)`` pair since #5894 — three excerpts of one paper
+ are one source."""
+ links = [
+ self._link(1, "https://a.example/x"),
+ self._link(2, "https://a.example/x"),
+ self._link(3, "https://a.example/x"),
+ self._link(4, "https://b.example/y"),
+ ]
+ strategy = self._make_strategy(all_links=links)
+ progress_updates = []
+ strategy.set_progress_callback(
+ lambda msg, pct, meta: progress_updates.append((msg, pct, meta))
+ )
+
+ strategy._finalize("q", "prose", 1, 0, [])
+
+ msg, _pct, meta = next(
+ u for u in progress_updates if u[2].get("phase") == "synthesis"
+ )
+ assert "reusing 2 accumulated sources" in msg
+ assert meta.get("accumulated_sources") == 2
+
def test_milestone_both_empty_emits_followup(self):
"""Both collectors empty -> emit only the fallback explanation milestone."""
strategy = self._make_strategy(all_links=[])
@@ -5612,3 +5712,583 @@ def test_sources_payload_does_not_mangle_ordinary_urls():
{"title": "P", "link": url, "url": url, "source": "library"}
)
assert collector.sources == [url], url
+
+
+# ---------------------------------------------------------------------------
+# #5894 — the citation dedup key is the (url, snippet) PAIR
+# ---------------------------------------------------------------------------
+
+
+def _collector(all_links=None):
+ from local_deep_research.advanced_search_system.strategies.langgraph_agent_strategy import (
+ SearchResultsCollector,
+ )
+
+ links = [] if all_links is None else all_links
+ return SearchResultsCollector(links), links
+
+
+class TestSnippetDedupKey:
+ """``_snippet_dedup_key`` — the second half of the citation key."""
+
+ @staticmethod
+ def _key(snippet):
+ from local_deep_research.advanced_search_system.strategies.langgraph_agent_strategy import (
+ _snippet_dedup_key,
+ )
+
+ return _snippet_dedup_key({"snippet": snippet})
+
+ def test_cosmetic_variants_of_one_passage_share_a_key(self):
+ """Whitespace, case and a trailing ellipsis are spellings of one
+ passage, not different passages.
+
+ Each variant is paired with a CONTROL that differs in actual
+ words: without it the test would pass just as well against a key
+ that collapses everything (e.g. a constant), which is the failure
+ mode a dedup key has.
+ """
+ base = "The quick brown fox jumps over the lazy dog"
+ for variant in (
+ " The quick brown fox\njumps over the lazy dog ",
+ "THE QUICK BROWN FOX JUMPS OVER THE LAZY DOG",
+ "The quick brown fox jumps over the lazy dog...",
+ "The quick brown fox jumps over the lazy dog\u2026",
+ "The quick brown fox jumps over the lazy dog.",
+ ):
+ assert self._key(variant) == self._key(base), variant
+
+ # Controls: a different passage must not share the key.
+ assert self._key("The quick brown fox jumps over the lazy cat") != (
+ self._key(base)
+ )
+ assert self._key(base + " And then it slept.") != self._key(base)
+
+ def test_two_passages_sharing_a_long_prefix_do_not_share_a_key(self):
+ """The digest covers the WHOLE passage, never a prefix.
+
+ A prefix comparison cannot tell "one passage truncated twice"
+ apart from "two passages that open the same way", and merging the
+ second case silently discards an excerpt — the outcome this whole
+ change exists to prevent. 400 shared characters, one differing
+ sentence at the end.
+ """
+ shared = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. " * 8
+ assert len(shared) > 400
+ assert self._key(shared + "The trial reported a 12% reduction.") != (
+ self._key(shared + "The trial reported no significant effect.")
+ )
+
+ def test_content_free_snippets_use_the_empty_sentinel(self):
+ """Anything with no content keys to ``""``.
+
+ The sentinel is returned deliberately rather than falling out of
+ ``blake2b("")``: an empty key means "no evidence here", and
+ ``_reuse_index`` routes it back to URL-only dedup so a
+ snippet-less repeat collapses exactly as it did before the
+ snippet joined the key.
+ """
+ from local_deep_research.advanced_search_system.strategies.langgraph_agent_strategy import (
+ _snippet_dedup_key,
+ )
+
+ for snippet in ("", " ", "\n\t ", "...", "\u2026", ". . ."):
+ assert self._key(snippet) == "", repr(snippet)
+ # Non-str values would be unhashable or meaningless in the key.
+ for snippet in (None, ["a", "b"], 7, {"x": 1}):
+ assert self._key(snippet) == "", repr(snippet)
+ # No snippet key at all, and the ``body`` fallback the renderer
+ # uses.
+ assert _snippet_dedup_key({}) == ""
+ assert _snippet_dedup_key({"body": "text"}) == self._key("text")
+ # An explicit empty ``snippet`` means "empty", so ``body`` is NOT
+ # consulted — matching ``_format_results``.
+ assert _snippet_dedup_key({"snippet": "", "body": "text"}) == ""
+ # A real passage still gets a real key.
+ assert self._key("a real passage") != ""
+
+
+class TestPairDedup:
+ """``add_results`` keyed on ``(canonical url, snippet)``."""
+
+ def test_a_second_excerpt_of_one_source_keeps_its_own_citation(self):
+ """The bug #5894 reports: a URL found again with a different
+ excerpt lost that excerpt entirely, because the collector reused
+ the first entry's index and never stored the occurrence.
+
+ Now it becomes its own entry with its own ``[N]``. The
+ same-excerpt repeat in the same batch is the control: it must
+ still collapse, or this test would pass against a collector with
+ no dedup at all.
+ """
+ collector, all_links = _collector()
+ _, indexed = collector.add_results(
+ [
+ {"title": "P", "link": "https://ex.test/p", "snippet": "one"},
+ {"title": "P", "link": "https://ex.test/p", "snippet": "two"},
+ {"title": "P", "link": "https://ex.test/p", "snippet": "one"},
+ ],
+ engine_name="web",
+ )
+
+ assert [r["index"] for r in indexed] == ["1", "2", "1"]
+ assert [e["snippet"] for e in all_links] == ["one", "two"]
+ # Each entry owns exactly one index, and each index one entry.
+ assert sorted(e["index"] for e in all_links) == ["1", "2"]
+ assert collector.find_by_index(1)["snippet"] == "one"
+ assert collector.find_by_index(2)["snippet"] == "two"
+
+ def test_a_cosmetic_repeat_collapses_but_a_real_one_does_not(self):
+ """End-to-end counterpart of the key-level test: normalisation
+ applies to the stored decision, not just to the helper."""
+ collector, all_links = _collector()
+ collector.add_results(
+ [{"title": "P", "link": "https://ex.test/p", "snippet": "One two"}]
+ )
+ collector.add_results(
+ [
+ {
+ "title": "P",
+ "link": "https://ex.test/p",
+ "snippet": " ONE\n\ntwo… ",
+ }
+ ]
+ )
+ assert len(all_links) == 1, all_links
+
+ collector.add_results(
+ [
+ {
+ "title": "P",
+ "link": "https://ex.test/p",
+ "snippet": "One two three",
+ }
+ ]
+ )
+ assert len(all_links) == 2, all_links
+
+ def test_a_snippet_less_repeat_still_collapses(self):
+ """No snippet means no evidence to preserve, so the occurrence
+ collapses on the URL alone — the behaviour every occurrence had
+ before this change. Without the sentinel a re-fetch of a cited
+ page with no excerpt would allocate a citation for nothing."""
+ collector, all_links = _collector()
+ collector.add_results(
+ [{"title": "P", "link": "https://ex.test/p", "snippet": "one"}]
+ )
+ _, indexed = collector.add_results(
+ [
+ {"title": "P", "link": "https://ex.test/p"},
+ {"title": "P", "link": "https://ex.test/p", "snippet": " "},
+ ]
+ )
+ assert [r["index"] for r in indexed] == ["1", "1"]
+ assert len(all_links) == 1
+ # Control: a real excerpt of the same URL is still kept.
+ collector.add_results(
+ [{"title": "P", "link": "https://ex.test/p", "snippet": "two"}]
+ )
+ assert len(all_links) == 2
+
+ def test_one_snippet_key_is_scoped_to_its_url(self):
+ """The key is the PAIR, so two sources may legitimately carry the
+ same text — a syndicated story, a shared abstract, boilerplate
+ reproduced across papers. A globally-scoped snippet key would let
+ whichever source was seen first suppress the other's copy, losing
+ an excerpt through the fix for losing excerpts."""
+ collector, all_links = _collector()
+ collector.add_results(
+ [
+ {"title": "A", "link": "https://a.test/x", "snippet": "shared"},
+ {"title": "B", "link": "https://b.test/y", "snippet": "shared"},
+ ]
+ )
+ assert len(all_links) == 2
+ assert [e["index"] for e in all_links] == ["1", "2"]
+
+ def test_excerpts_survive_a_subsection_reset(self):
+ """``reset()`` clears the per-subsection state; the dedup maps
+ deliberately persist, so a source re-cited in a later subsection
+ keeps its number and a genuinely new excerpt of it still gets
+ one."""
+ collector, all_links = _collector()
+ collector.add_results(
+ [{"title": "P", "link": "https://ex.test/p", "snippet": "s1"}]
+ )
+ collector.reset()
+
+ _, indexed = collector.add_results(
+ [
+ # Same excerpt as §1 — still [1], no second entry.
+ {"title": "P", "link": "https://ex.test/p", "snippet": "s1"},
+ # New excerpt in this subsection — its own citation.
+ {"title": "P", "link": "https://ex.test/p", "snippet": "s2"},
+ ]
+ )
+ assert [r["index"] for r in indexed] == ["1", "2"]
+ assert len(all_links) == 2
+ # One source for this subsection, not two.
+ assert collector.sources == ["https://ex.test/p"]
+
+ def test_library_chunks_of_one_document_stay_one_source(self):
+ """Two chunks of one library document are two excerpts of ONE
+ source: two citations, one ## Sources line, each citation keeping
+ its own ``#chunk-<n>`` anchor so the reader lands on the text the
+ excerpt came from.
+
+ A repeat of the SAME chunk with the same excerpt still collapses.
+ """
+ from local_deep_research.utilities.search_utilities import (
+ count_distinct_sources,
+ )
+
+ collector, all_links = _collector()
+ doc = "550e8400-e29b-41d4-a716-446655440000"
+
+ def chunk(n, snippet):
+ return {
+ "title": "Doc",
+ "link": f"/library/document/{doc}",
+ "source": "library",
+ "snippet": snippet,
+ "metadata": {"doc_id": doc, "chunk_index": n},
+ }
+
+ _, indexed = collector.add_results(
+ [chunk(0, "first passage"), chunk(3, "third passage")]
+ )
+ assert [r["index"] for r in indexed] == ["1", "2"]
+ _, again = collector.add_results([chunk(0, "first passage")])
+ assert again[0]["index"] == "1"
+
+ assert len(all_links) == 2
+ assert all_links[0]["link"].endswith("#chunk-0")
+ assert all_links[1]["link"].endswith("#chunk-3")
+ # ...but one document, one bibliography line, both numbers on it.
+ assert count_distinct_sources(all_links) == 1
+ rendered = format_links_to_markdown(all_links)
+ assert rendered.count("URL:") == 1
+ assert "[1, 2]" in rendered
+
+ def test_agent_facing_block_is_unchanged(self):
+ """``_format_results`` is not touched by this change: for a given
+ input list its output must be byte-identical to ``main``.
+
+ The literal below was captured from the function on ``main``; the
+ source line that builds it is unchanged in this PR.
+ """
+ from local_deep_research.advanced_search_system.strategies.langgraph_agent_strategy import (
+ _format_results,
+ )
+
+ results = [
+ {
+ "index": "1",
+ "title": "Paper",
+ "link": "https://ex.test/p",
+ "snippet": "first excerpt",
+ },
+ {
+ "index": "2",
+ "title": "Paper",
+ "link": "https://ex.test/p",
+ "snippet": "second excerpt",
+ },
+ {
+ "index": "3",
+ "title": "Other",
+ "link": "https://o.test/q",
+ "body": "b",
+ },
+ {"title": "No index", "url": "https://n.test/r"},
+ ]
+
+ assert _format_results(results, 7) == (
+ "[1] Paper (https://ex.test/p)\nfirst excerpt\n\n"
+ "[2] Paper (https://ex.test/p)\nsecond excerpt\n\n"
+ "[3] Other (https://o.test/q)\nb\n\n"
+ "[11] No index (https://n.test/r)\n"
+ )
+
+ def test_a_source_accumulates_every_distinct_excerpt(self):
+ """No cap: a source may own as many citations as it has distinct
+ excerpts.
+
+ An earlier revision of #5894 bounded this at three and swept the
+ rest onto the source's FIRST citation. That showed the model
+ excerpt 4's text under ``[1]``; ``[1]`` resolves to excerpt 1, so
+ the citation pointed at evidence nobody read — silently. A live
+ measurement (32 queries, 630 results, 573 URLs, real Brave SERP)
+ found 84.2% of same-URL query pairs carry DIFFERENT snippet text,
+ median character similarity 0.18, so the discarded tail is
+ distinct evidence, which is the exact thing this collector exists
+ to preserve.
+
+ The count is bounded anyway by the number of queries run
+ (``iterations`` x ``questions_per_iteration``).
+ """
+ from local_deep_research.utilities.search_utilities import (
+ count_distinct_sources,
+ )
+
+ collector, all_links = _collector()
+ excerpts = [f"distinct passage number {n}" for n in range(6)]
+ _, indexed = collector.add_results(
+ [
+ {"title": "P", "link": "https://ex.test/p", "snippet": e}
+ for e in excerpts
+ ],
+ engine_name="web",
+ )
+
+ # Six distinct excerpts, six entries, six citation indices — and
+ # each index resolves to the excerpt it was allocated for.
+ assert len(all_links) == 6
+ assert [r["index"] for r in indexed] == ["1", "2", "3", "4", "5", "6"]
+ assert [e["snippet"] for e in all_links] == excerpts
+ for idx, excerpt in zip(
+ [r["index"] for r in indexed], excerpts, strict=True
+ ):
+ assert collector.find_by_index(int(idx))["snippet"] == excerpt
+
+ # Still ONE source: the entries are excerpts of it, not sources.
+ assert count_distinct_sources(all_links) == 1
+
+ # And a repeat of an excerpt that already has a citation still
+ # collapses onto THAT citation, not onto a new one.
+ _, repeat = collector.add_results(
+ [
+ {
+ "title": "P",
+ "link": "https://ex.test/p",
+ "snippet": "distinct passage number 3",
+ }
+ ],
+ engine_name="web",
+ )
+ assert [r["index"] for r in repeat] == ["4"]
+ assert len(all_links) == 6
+
+ def test_concurrent_add_results_allocate_one_entry_per_pair(self):
+ """Thread safety, with a critical section wide enough for the lock
+ to matter.
+
+ ``_reuse_index`` is monkeypatched to sleep AFTER reading the dedup
+ maps and BEFORE the caller allocates, which is exactly the window
+ the lock closes. Without mutual exclusion several threads read
+ "not present" for the same pair and each appends, so the shared
+ excerpt is stored more than once and two entries claim one
+ citation index. Verified by replacing ``collector._lock`` with a
+ fresh unshared lock: this test then fails. A plain concurrency
+ smoke test does not — the critical section is short enough that
+ the GIL serialises it by accident.
+ """
+ import time
+
+ from local_deep_research.advanced_search_system.strategies.langgraph_agent_strategy import (
+ SearchResultsCollector,
+ )
+
+ real_reuse = SearchResultsCollector._reuse_index
+
+ def slow_reuse(self, key, snippet_key):
+ found = real_reuse(self, key, snippet_key)
+ time.sleep(0.002)
+ return found
+
+ collector, all_links = _collector()
+ url = "https://ex.test/paper"
+ threads = 8
+ rounds = 3
+ errors = []
+
+ def worker(tid):
+ try:
+ for r in range(rounds):
+ collector.add_results(
+ [
+ # Distinct per (thread, round): must all survive.
+ {
+ "title": "P",
+ "link": url,
+ "snippet": f"excerpt {tid}-{r}",
+ },
+ # Shared by every thread: must collapse to ONE.
+ {"title": "P", "link": url, "snippet": "shared"},
+ ],
+ engine_name="web",
+ )
+ except Exception as exc: # pragma: no cover - failure path
+ errors.append(exc)
+
+ with patch.object(SearchResultsCollector, "_reuse_index", slow_reuse):
+ workers = [
+ threading.Thread(target=worker, args=(t,))
+ for t in range(threads)
+ ]
+ for t in workers:
+ t.start()
+ for t in workers:
+ t.join()
+
+ assert not errors, errors
+ expected = threads * rounds + 1
+ assert len(all_links) == expected, len(all_links)
+ assert [e["snippet"] for e in all_links].count("shared") == 1
+ indices = [e["index"] for e in all_links]
+ assert len(set(indices)) == expected, "an index was allocated twice"
+ assert sorted(int(i) for i in indices) == list(range(1, expected + 1))
+
+
+class TestSourcesBlockStaysCoherent:
+ """The consequences of several entries per source, at the renderer."""
+
+ def test_a_second_excerpt_is_one_sources_line_with_both_numbers(self):
+ from local_deep_research.utilities.search_utilities import (
+ count_distinct_sources,
+ )
+
+ collector, all_links = _collector()
+ collector.add_results(
+ [
+ {"title": "P", "link": "https://ex.test/p", "snippet": "one"},
+ {"title": "O", "link": "https://o.test/q", "snippet": "q"},
+ ]
+ )
+ collector.add_results(
+ [{"title": "P", "link": "https://ex.test/p", "snippet": "two"}]
+ )
+
+ rendered = format_links_to_markdown(all_links)
+
+ assert rendered.count("URL:") == 2
+ assert "[1, 3] P" in rendered
+ assert "[2] O" in rendered
+ assert count_distinct_sources(all_links) == 2
+
+ def test_divergent_url_field_does_not_split_the_sources_line(self):
+ """The collector keys a citation on ``link``; the bibliography used
+ to group on ``url or link`` — ``url`` FIRST.
+
+ While one entry per source survived, the disagreement could not
+ show. With a second excerpt of the same source in the list, a
+ result carrying BOTH fields with different values landed in a
+ render group of its own and gave one source two ## Sources lines
+ under different numbers. ``source_url_field`` is now the single
+ definition of which field identifies a source, so the two agree.
+ """
+ collector, all_links = _collector()
+ collector.add_results(
+ [
+ {
+ "title": "P",
+ "link": "https://ex.test/p",
+ "snippet": "one",
+ }
+ ]
+ )
+ collector.add_results(
+ [
+ {
+ "title": "P",
+ "link": "https://ex.test/p",
+ # A DIFFERENT url on the same citation.
+ "url": "https://elsewhere.test/mirror",
+ "snippet": "two",
+ }
+ ]
+ )
+
+ assert len(all_links) == 2
+ rendered = format_links_to_markdown(all_links)
+ assert rendered.count("URL:") == 1, rendered
+ assert "https://elsewhere.test/mirror" not in rendered
+ assert "[1, 2]" in rendered
+
+ def test_second_excerpt_does_not_repoint_the_citation_hyperlink(self):
+ """Regression pin for the withdrawn first attempt at #5894.
+
+ That version appended a second entry while REUSING the first
+ entry's index, making index -> entry one-to-many.
+ ``CitationFormatter.apply_inline_hyperlinks`` builds its
+ index -> url map LAST-WINS and does not canonicalise, so the later
+ spelling — credentials and all — won, and ``[[1]]`` in the report
+ (and the row written to ``research_resources``) pointed at
+ ``https://admin:[email protected]/paper``.
+
+ With one index per entry the map is 1:1 again and an occurrence
+ can only ever describe its own citation. This test passes by
+ construction under the current design; it is kept to pin that
+ property against any future change that reintroduces index reuse.
+
+ It does NOT claim the collector sanitises a URL spelling — a
+ citation renders whatever URL its own entry carries, here and on
+ ``main`` alike.
+ """
+ from local_deep_research.text_optimization.citation_formatter import (
+ CitationFormatter,
+ CitationMode,
+ )
+
+ collector, all_links = _collector()
+ collector.add_results(
+ [
+ {
+ "title": "Paper",
+ "link": "https://ex.test/paper",
+ "snippet": "first excerpt",
+ }
+ ]
+ )
+ collector.add_results(
+ [
+ {
+ "title": "Paper",
+ "link": "https://admin:[email protected]/paper",
+ "snippet": "second, different excerpt",
+ }
+ ]
+ )
+
+ formatter = CitationFormatter(mode=CitationMode.NUMBER_HYPERLINKS)
+ rendered = formatter.apply_inline_hyperlinks(
+ "The trial reported a reduction [1].", all_links
+ )
+
+ assert rendered == (
+ "The trial reported a reduction [[1]](https://ex.test/paper)."
+ )
+ # The bibliography canonicalises userinfo away for both entries.
+ assert "hunter2" not in format_links_to_markdown(all_links)
+
+ def test_source_counts_count_sources_not_entries(self):
+ """``len(all_links_of_system)`` is no longer a source count."""
+ from local_deep_research.advanced_search_system.strategies.langgraph_agent_strategy import (
+ LangGraphAgentStrategy,
+ )
+ from local_deep_research.utilities.search_utilities import (
+ count_distinct_sources,
+ )
+
+ all_links = []
+ strategy = LangGraphAgentStrategy(
+ model=MagicMock(),
+ search=MagicMock(),
+ all_links_of_system=all_links,
+ settings_snapshot={"search.tool": {"value": "searxng"}},
+ )
+ strategy._tool_names = ["web_search"]
+ strategy.collector.add_results(
+ [
+ {"title": "P", "link": "https://ex.test/p", "snippet": "one"},
+ {"title": "P", "link": "https://ex.test/p", "snippet": "two"},
+ {"title": "P", "link": "https://ex.test/p", "snippet": "three"},
+ {"title": "O", "link": "https://o.test/q", "snippet": "q"},
+ ]
+ )
+
+ assert len(all_links) == 4
+ assert count_distinct_sources(all_links) == 2
+ assert strategy._heartbeat_message(2).startswith(
+ "Step 2 · 2 sources gathered · "
+ )
diff --git a/tests/test_report_generator_coverage.py b/tests/test_report_generator_coverage.py
index 373547597e..6db59fca77 100644
--- a/tests/test_report_generator_coverage.py
+++ b/tests/test_report_generator_coverage.py
@@ -282,14 +282,23 @@ class TestFormatFinalReportDetailed:
report = generator._format_final_report(sections, structure, "q")
assert report["metadata"]["searches_per_section"] == 7
- def test_metadata_initial_sources_counts_links(
+ def test_metadata_initial_sources_counts_distinct_sources(
self, generator, _patch_importlib
):
- """Metadata initial_sources equals len(all_links_of_system)."""
+ """Metadata initial_sources counts distinct SOURCES.
+
+ Not ``len(all_links_of_system)``. That list holds one entry per
+ distinct ``(url, snippet)`` pair under the LangGraph strategy and
+ raw un-deduped engine dicts under the others, so its length
+ counts occurrences. The number is reported as "sources", so it
+ must group the way the ``## Sources`` block does — here four
+ entries over three URLs read as three.
+ """
generator.search_system.all_links_of_system = [
- {"link": "a"},
- {"link": "b"},
- {"link": "c"},
+ {"link": "https://a.test/x", "snippet": "one"},
+ {"link": "https://a.test/x", "snippet": "two"},
+ {"link": "https://b.test/y"},
+ {"link": "https://c.test/z"},
]
structure = []
sections = {}
diff --git a/tests/utilities/test_search_utilities_formatting.py b/tests/utilities/test_search_utilities_formatting.py
index 4b5b0791c3..4b3356bd5d 100644
--- a/tests/utilities/test_search_utilities_formatting.py
+++ b/tests/utilities/test_search_utilities_formatting.py
@@ -1260,3 +1260,119 @@ def test_recorded_anchor_for_ANOTHER_document_is_not_rendered():
]
)
assert own in out_own, out_own
+
+
+class TestCountDistinctSources:
+ """``count_distinct_sources`` — the "how many sources" number.
+
+ ``len(all_links_of_system)`` counts occurrences: the LangGraph
+ collector stores one entry per distinct ``(url, snippet)`` pair since
+ #5894, and ``source_based`` / ``focused_iteration`` /
+ ``topic_organization`` extend the list with raw engine dicts and no
+ URL dedup at all.
+ """
+
+ def test_counts_the_lines_the_bibliography_renders(self):
+ from local_deep_research.utilities.search_utilities import (
+ count_distinct_sources,
+ format_links_to_markdown,
+ )
+
+ links = [
+ {"title": "P", "link": "https://ex.test/p", "index": "1"},
+ {"title": "P", "link": "https://ex.test/p", "index": "2"},
+ {"title": "O", "link": "https://o.test/q", "index": "3"},
+ ]
+
+ assert count_distinct_sources(links) == 2
+ # The two must agree — that is the whole point of sharing the
+ # grouping expression.
+ assert format_links_to_markdown(links).count("URL:") == 2
+
+ def test_groups_by_the_canonical_key_like_the_renderer(self):
+ from local_deep_research.utilities.search_utilities import (
+ count_distinct_sources,
+ )
+
+ links = [
+ {"link": "https://ex.test/p", "index": "1"},
+ {"link": "https://EX.test/p/?utm_source=x", "index": "2"},
+ {"link": "https://user:[email protected]/p#frag", "index": "3"},
+ ]
+ assert count_distinct_sources(links) == 1
+ # Control: a genuinely different URL is a second source.
+ assert (
+ count_distinct_sources(links + [{"link": "https://o.test/q"}]) == 2
+ )
+
+ def test_skips_entries_the_renderer_also_skips(self):
+ from local_deep_research.utilities.search_utilities import (
+ count_distinct_sources,
+ )
+
+ assert count_distinct_sources([]) == 0
+ assert count_distinct_sources(None) == 0
+ assert (
+ count_distinct_sources(
+ [
+ "not a dict",
+ {"link": None},
+ {"link": ["a"]},
+ {"title": "no url at all"},
+ {"link": ""},
+ ]
+ )
+ == 0
+ )
+
+
+class TestSourceUrlField:
+ """Which field identifies a source, for grouping.
+
+ ``SearchResultsCollector`` keys every citation on the entry's
+ ``link``. The bibliography used to group on ``url or link`` —
+ ``url`` first — and while at most one entry per source existed the
+ disagreement could not show. It can now, so both read
+ ``source_url_field``.
+ """
+
+ def test_link_wins_over_a_divergent_url(self):
+ from local_deep_research.utilities.search_utilities import (
+ source_url_field,
+ )
+
+ assert (
+ source_url_field(
+ {"link": "https://a.test/x", "url": "https://b.test/y"}
+ )
+ == "https://a.test/x"
+ )
+ # ...and ``url`` still serves the raw engine dicts that set only it.
+ assert source_url_field({"url": "https://b.test/y"}) == (
+ "https://b.test/y"
+ )
+ assert source_url_field({"link": "", "url": "https://b.test/y"}) == (
+ "https://b.test/y"
+ )
+ assert source_url_field({}) == ""
+
+ def test_two_entries_of_one_source_render_one_line_despite_a_divergent_url(
+ self,
+ ):
+ from local_deep_research.utilities.search_utilities import (
+ format_links_to_markdown,
+ )
+
+ links = [
+ {"title": "P", "link": "https://ex.test/p", "index": "1"},
+ {
+ "title": "P",
+ "link": "https://ex.test/p",
+ "url": "https://elsewhere.test/mirror",
+ "index": "2",
+ },
+ ]
+ rendered = format_links_to_markdown(links)
+ assert rendered.count("URL:") == 1, rendered
+ assert "[1, 2]" in rendered
+ assert "elsewhere.test" not in rendered
diff --git a/tests/web/services/test_report_assembly_service.py b/tests/web/services/test_report_assembly_service.py
index cb0e9874f1..9250362465 100644
--- a/tests/web/services/test_report_assembly_service.py
+++ b/tests/web/services/test_report_assembly_service.py
@@ -382,6 +382,67 @@ class TestGetResearchSourceLinks:
links = get_research_source_links(research.id, db_session)
assert [link["url"] for link in links] == ["https://yes.com"]
+ def test_limit_counts_distinct_sources_not_rows(self, db_session):
+ """Three rows for one URL must not fill a top-3 card with one source.
+
+ A strategy stores one ``research_resources`` row per piece of
+ evidence, so one URL legitimately appears several times with
+ different snippets (#5894). Before the dedup this returned the
+ SAME url three times — zero diversity in a "top 3 sources" card.
+ """
+ research = _mk_research(db_session)
+ for i in range(3):
+ _mk_resource(
+ db_session,
+ research.id,
+ url="https://dup.com/page",
+ title=f"Dup {i}",
+ index="1",
+ )
+ for i, url in enumerate(["https://b.com", "https://c.com"], start=2):
+ _mk_resource(
+ db_session, research.id, url=url, title=f"T{i}", index=str(i)
+ )
+
+ links = get_research_source_links(research.id, db_session, limit=3)
+ assert [link["url"] for link in links] == [
+ "https://dup.com/page",
+ "https://b.com",
+ "https://c.com",
+ ]
+ # First occurrence wins, so the title stays the one the earliest
+ # row carried rather than the last duplicate's.
+ assert links[0]["title"] == "Dup 0"
+
+ def test_dedup_uses_the_canonical_key_the_bibliography_groups_by(
+ self, db_session
+ ):
+ """Noisy spellings of one URL are one source, exactly as in Sources.
+
+ Grouping on the raw string instead would let a ``utm_``-tagged or
+ fragment-bearing spelling of a page count as a second source here
+ while ``format_links_to_markdown`` renders it as one line.
+ """
+ research = _mk_research(db_session)
+ for i, url in enumerate(
+ [
+ "https://a.com/page",
+ "https://A.com/page/?utm_source=news",
+ "https://a.com/page#section",
+ "https://d.com/page",
+ ],
+ start=1,
+ ):
+ _mk_resource(
+ db_session, research.id, url=url, title=f"T{i}", index=str(i)
+ )
+
+ links = get_research_source_links(research.id, db_session, limit=3)
+ assert [link["url"] for link in links] == [
+ "https://a.com/page",
+ "https://d.com/page",
+ ]
+
def test_falls_back_to_domain_when_title_missing(self, db_session):
research = _mk_research(db_session)
r = ResearchResource(
@@ -424,6 +485,37 @@ class TestGetResearchSourceLinksBatch:
batch = get_research_source_links_batch(["empty-r"], db_session)
assert batch == {"empty-r": []}
+ def test_limit_counts_distinct_sources_per_research(self, db_session):
+ """Batch dedups like its single-research sibling, and per research.
+
+ The seen-set must be scoped to one research id: a URL cited by two
+ different researches has to survive in both buckets.
+ """
+ r1 = _mk_research(db_session, id="dr1")
+ r2 = _mk_research(db_session, id="dr2")
+ for i in range(3):
+ _mk_resource(
+ db_session, r1.id, url="https://dup.com/p", title=f"D{i}"
+ )
+ _mk_resource(db_session, r1.id, url="https://other.com/p", title="O")
+ _mk_resource(db_session, r2.id, url="https://dup.com/p", title="R2")
+
+ batch = get_research_source_links_batch(["dr1", "dr2"], db_session)
+ assert [link["url"] for link in batch["dr1"]] == [
+ "https://dup.com/p",
+ "https://other.com/p",
+ ]
+ # Not suppressed by dr1 having already seen this URL.
+ assert [link["url"] for link in batch["dr2"]] == ["https://dup.com/p"]
+
+ def test_unlimited_still_dedups(self, db_session):
+ """``limit=None`` (the report API) uncaps the count, not the dedup."""
+ r = _mk_research(db_session, id="unl")
+ for i in range(4):
+ _mk_resource(db_session, r.id, url="https://one.com/p", title="X")
+ batch = get_research_source_links_batch(["unl"], db_session, limit=None)
+ assert [link["url"] for link in batch["unl"]] == ["https://one.com/p"]
+
def test_respects_limit_per_research(self, db_session):
r = _mk_research(db_session)
for i, url in enumerate(