Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(citations): accept pre-comma whitespace in source groups (#7713)

Chanhyo Jungd7c8d8158968
diff --git a/changelog.d/7666.bugfix.md b/changelog.d/7666.bugfix.md
new file mode 100644
index 0000000000..f0718416e3
--- /dev/null
+++ b/changelog.d/7666.bugfix.md
@@ -0,0 +1 @@
+Fix source groups with spaces before commas in citation links, collections, bibliographies, and RIS exports.
diff --git a/src/local_deep_research/text_optimization/citation_formatter.py b/src/local_deep_research/text_optimization/citation_formatter.py
index 667b8dd30d..6418af87ad 100644
--- a/src/local_deep_research/text_optimization/citation_formatter.py
+++ b/src/local_deep_research/text_optimization/citation_formatter.py
@@ -325,7 +325,7 @@ def strip_appended_sources_sentinel(content: str) -> str:
_BIB_SOURCES_PATTERN = re.compile(
- r"^\[(\d+(?:,\s*\d+)*)\]\s*(.+?)(?:\n\s*URL:\s*(.+?))?$",
+ r"^\[(\d+(?:\s*,\s*\d+)*)\]\s*(.+?)(?:\n\s*URL:\s*(.+?))?$",
re.MULTILINE,
)
@@ -567,7 +567,7 @@ class CitationFormatter:
# Also match "Source X" or "source X" patterns
self.source_word_pattern = re.compile(r"\b[Ss]ource\s+(\d+)\b")
self.sources_pattern = re.compile(
- r"^\[(\d+(?:,\s*\d+)*)\]\s*(.+?)(?:\n\s*URL:\s*(.+?))?$",
+ r"^\[(\d+(?:\s*,\s*\d+)*)\]\s*(.+?)(?:\n\s*URL:\s*(.+?))?$",
re.MULTILINE,
)
@@ -1034,7 +1034,7 @@ class CitationFormatter:
# library hits (emitted by ``utilities/search_utilities.format_links_to_markdown``).
# Read one line at a time: overlapping multiline whitespace patterns
# can repeatedly rescan an entire run of blank lines.
- _collection_header_pattern = re.compile(r"^\[(\d+(?:,\s*\d+)*)\]")
+ _collection_header_pattern = re.compile(r"^\[(\d+(?:\s*,\s*\d+)*)\]")
def _parse_collections(self, sources_content: str) -> Dict[str, str]:
"""Extract ``{citation_num: collection_name}`` from a sources
@@ -2474,8 +2474,8 @@ class RISExporter:
# is not this fix — it is single-line and section-unaware, so
# wiring it in would lose properties 1 and 3.
source_entry_pattern = re.compile(
- r"^[\[【](\d+(?:,\s*\d+)*)[\]】]\s*(.+?)"
- r"(?=^[\[【]\d+(?:,\s*\d+)*[\]】]|\Z)",
+ r"^[\[【](\d+(?:\s*,\s*\d+)*)[\]】]\s*(.+?)"
+ r"(?=^[\[【]\d+(?:\s*,\s*\d+)*[\]】]|\Z)",
re.MULTILINE | re.DOTALL,
)
diff --git a/tests/text_optimization/test_citation_formatter.py b/tests/text_optimization/test_citation_formatter.py
index 7c0fe1be9e..f56455ee19 100644
--- a/tests/text_optimization/test_citation_formatter.py
+++ b/tests/text_optimization/test_citation_formatter.py
@@ -2597,3 +2597,104 @@ class TestUrlClassifierIsNormalImport:
assert cf.URLClassifier is URLClassifier
assert cf.URLType is URLType
+
+
[email protected]("group", ["2,003", "2, 003", "2 ,003", "2 , 003"])
+class TestSpacedSourceGroups:
+ def test_bibliography(self, group):
+ from local_deep_research.text_optimization.citation_formatter import (
+ _collect_bibliography_entries,
+ )
+
+ sources = (
+ f"[{group}] Earlier\n URL: https://example.com/old\n"
+ f"[{group}] Shared\n URL: https://example.org/shared\n"
+ f"[{group}] URL-less prose\n"
+ )
+ expected = {
+ "2": ("Shared", "https://example.org/shared"),
+ "003": ("Shared", "https://example.org/shared"),
+ }
+ assert _collect_bibliography_entries(sources) == expected
+ content = f"See [2] and [003].\n\n## Sources\n{sources}"
+ assert _collect_bibliography_entries(content) == expected
+ exporter = QuartoExporter()
+ bib = exporter._generate_bibliography(content)
+ for index in expected:
+ record = bib.split(f"@misc{{ref{index},", 1)[1].split("@misc", 1)[0]
+ assert "Shared" in record
+ assert "https://example.org/shared" in record
+ exported = exporter.export_to_quarto(content)
+ assert bib in exported
+ assert "@ref2" in exported
+ assert "@ref003" in exported
+
+ def test_markdown(self, group):
+ sources = f"[{group}] Shared\n URL: https://example.org/shared\n"
+ formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS)
+ assert formatter._parse_sources(sources) == {
+ "2": ("Shared", "https://example.org/shared"),
+ "003": ("Shared", "https://example.org/shared"),
+ }
+ result = formatter.format_document(
+ f"See [2] then [003].\n\n## Sources\n{sources}"
+ )
+ assert (
+ "See [[2]](https://example.org/shared) then "
+ "[[003]](https://example.org/shared)."
+ ) in result
+
+ def test_collections(self, group):
+ sources = (
+ "Collection: Unattached\n"
+ f"[{group}] Shared\n Collection: Intended\n"
+ " Collection: Ignored second\n"
+ "[4] No collection\n[5] Next\n Collection: Next\n"
+ "[6] Reset\n[invalid] Bad\n Collection: Detached\n"
+ )
+ assert CitationFormatter()._parse_collections(sources) == {
+ "2": "Intended",
+ "003": "Intended",
+ "5": "Next",
+ }
+
+
+ "brackets", ["[]", "【】"], ids=["ascii", "lenticular"]
+)
[email protected]("group", ["2,003", "2, 003", "2 ,003", "2 , 003"])
+def test_spaced_ris_adjacent_records(brackets, group):
+ opening, closing = brackets
+ content = (
+ "## Sources\n"
+ "[1] Before\n URL: https://example.com/before\n"
+ " DOI: 10.1234/before\n"
+ f"{opening}{group}{closing} Shared\n"
+ " by Example Author.\n Published in Example Journal 2020\n"
+ " URL: https://example.org/shared\n DOI: 10.1234/shared\n"
+ "[4] After\n URL: https://example.com/after\n"
+ " DOI: 10.1234/after\n"
+ "[4] Final\n URL: https://example.org/final\n"
+ " DOI: 10.1234/final\n"
+ "## ALL SOURCES\n[9] Excluded\n URL: https://example.com/no\n"
+ )
+ result = RISExporter().export_to_ris(content)
+ records = [record for record in result.split("ER -") if record.strip()]
+ assert "UR - https://example.com/before\n" in records[0]
+ assert "DO - 10.1234/before\n" in records[0]
+ assert len(records) == 4
+ assert [
+ line for line in result.splitlines() if line.startswith("ID -")
+ ] == ["ID - ref1", "ID - ref2", "ID - ref003", "ID - ref4"]
+ for record, title, suffix, host in zip(
+ records,
+ ["Before", "Shared", "Shared", "Final"],
+ ["before", "shared", "shared", "final"],
+ ["example.com", "example.org", "example.org", "example.org"],
+ ):
+ assert f"TI - {title}\n" in record
+ assert f"UR - https://{host}/{suffix}\n" in record
+ assert f"DO - 10.1234/{suffix}\n" in record
+ for record in records[1:3]:
+ assert "AU - Example Author\n" in record
+ assert "PY - 2020\n" in record