~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(citations): accept pre-comma whitespace in source groups (#7713)
Chanhyo Jungd7c8d8158968
diff --git a/changelog.d/7666.bugfix.md b/changelog.d/7666.bugfix.mdnew file mode 100644index 0000000000..f0718416e3--- /dev/null+++ b/changelog.d/7666.bugfix.md@@ -0,0 +1 @@+Fix source groups with spaces before commas in citation links, collections, bibliographies, and RIS exports.diff --git a/src/local_deep_research/text_optimization/citation_formatter.py b/src/local_deep_research/text_optimization/citation_formatter.pyindex 667b8dd30d..6418af87ad 100644--- a/src/local_deep_research/text_optimization/citation_formatter.py+++ b/src/local_deep_research/text_optimization/citation_formatter.py@@ -325,7 +325,7 @@ def strip_appended_sources_sentinel(content: str) -> str:_BIB_SOURCES_PATTERN = re.compile(- r"^\[(\d+(?:,\s*\d+)*)\]\s*(.+?)(?:\n\s*URL:\s*(.+?))?$",+ r"^\[(\d+(?:\s*,\s*\d+)*)\]\s*(.+?)(?:\n\s*URL:\s*(.+?))?$",re.MULTILINE,)@@ -567,7 +567,7 @@ class CitationFormatter:# Also match "Source X" or "source X" patternsself.source_word_pattern = re.compile(r"\b[Ss]ource\s+(\d+)\b")self.sources_pattern = re.compile(- r"^\[(\d+(?:,\s*\d+)*)\]\s*(.+?)(?:\n\s*URL:\s*(.+?))?$",+ r"^\[(\d+(?:\s*,\s*\d+)*)\]\s*(.+?)(?:\n\s*URL:\s*(.+?))?$",re.MULTILINE,)@@ -1034,7 +1034,7 @@ class CitationFormatter:# library hits (emitted by ``utilities/search_utilities.format_links_to_markdown``).# Read one line at a time: overlapping multiline whitespace patterns# can repeatedly rescan an entire run of blank lines.- _collection_header_pattern = re.compile(r"^\[(\d+(?:,\s*\d+)*)\]")+ _collection_header_pattern = re.compile(r"^\[(\d+(?:\s*,\s*\d+)*)\]")def _parse_collections(self, sources_content: str) -> Dict[str, str]:"""Extract ``{citation_num: collection_name}`` from a sources@@ -2474,8 +2474,8 @@ class RISExporter:# is not this fix — it is single-line and section-unaware, so# wiring it in would lose properties 1 and 3.source_entry_pattern = re.compile(- r"^[\[【](\d+(?:,\s*\d+)*)[\]】]\s*(.+?)"- r"(?=^[\[【]\d+(?:,\s*\d+)*[\]】]|\Z)",+ r"^[\[【](\d+(?:\s*,\s*\d+)*)[\]】]\s*(.+?)"+ r"(?=^[\[【]\d+(?:\s*,\s*\d+)*[\]】]|\Z)",re.MULTILINE | re.DOTALL,)diff --git a/tests/text_optimization/test_citation_formatter.py b/tests/text_optimization/test_citation_formatter.pyindex 7c0fe1be9e..f56455ee19 100644--- a/tests/text_optimization/test_citation_formatter.py+++ b/tests/text_optimization/test_citation_formatter.py@@ -2597,3 +2597,104 @@ class TestUrlClassifierIsNormalImport:assert cf.URLClassifier is URLClassifierassert cf.URLType is URLType++[email protected]("group", ["2,003", "2, 003", "2 ,003", "2 , 003"])+class TestSpacedSourceGroups:+ def test_bibliography(self, group):+ from local_deep_research.text_optimization.citation_formatter import (+ _collect_bibliography_entries,+ )++ sources = (+ f"[{group}] Earlier\n URL: https://example.com/old\n"+ f"[{group}] Shared\n URL: https://example.org/shared\n"+ f"[{group}] URL-less prose\n"+ )+ expected = {+ "2": ("Shared", "https://example.org/shared"),+ "003": ("Shared", "https://example.org/shared"),+ }+ assert _collect_bibliography_entries(sources) == expected+ content = f"See [2] and [003].\n\n## Sources\n{sources}"+ assert _collect_bibliography_entries(content) == expected+ exporter = QuartoExporter()+ bib = exporter._generate_bibliography(content)+ for index in expected:+ record = bib.split(f"@misc{{ref{index},", 1)[1].split("@misc", 1)[0]+ assert "Shared" in record+ assert "https://example.org/shared" in record+ exported = exporter.export_to_quarto(content)+ assert bib in exported+ assert "@ref2" in exported+ assert "@ref003" in exported++ def test_markdown(self, group):+ sources = f"[{group}] Shared\n URL: https://example.org/shared\n"+ formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS)+ assert formatter._parse_sources(sources) == {+ "2": ("Shared", "https://example.org/shared"),+ "003": ("Shared", "https://example.org/shared"),+ }+ result = formatter.format_document(+ f"See [2] then [003].\n\n## Sources\n{sources}"+ )+ assert (+ "See [[2]](https://example.org/shared) then "+ "[[003]](https://example.org/shared)."+ ) in result++ def test_collections(self, group):+ sources = (+ "Collection: Unattached\n"+ f"[{group}] Shared\n Collection: Intended\n"+ " Collection: Ignored second\n"+ "[4] No collection\n[5] Next\n Collection: Next\n"+ "[6] Reset\n[invalid] Bad\n Collection: Detached\n"+ )+ assert CitationFormatter()._parse_collections(sources) == {+ "2": "Intended",+ "003": "Intended",+ "5": "Next",+ }+++ "brackets", ["[]", "【】"], ids=["ascii", "lenticular"]+)[email protected]("group", ["2,003", "2, 003", "2 ,003", "2 , 003"])+def test_spaced_ris_adjacent_records(brackets, group):+ opening, closing = brackets+ content = (+ "## Sources\n"+ "[1] Before\n URL: https://example.com/before\n"+ " DOI: 10.1234/before\n"+ f"{opening}{group}{closing} Shared\n"+ " by Example Author.\n Published in Example Journal 2020\n"+ " URL: https://example.org/shared\n DOI: 10.1234/shared\n"+ "[4] After\n URL: https://example.com/after\n"+ " DOI: 10.1234/after\n"+ "[4] Final\n URL: https://example.org/final\n"+ " DOI: 10.1234/final\n"+ "## ALL SOURCES\n[9] Excluded\n URL: https://example.com/no\n"+ )+ result = RISExporter().export_to_ris(content)+ records = [record for record in result.split("ER -") if record.strip()]+ assert "UR - https://example.com/before\n" in records[0]+ assert "DO - 10.1234/before\n" in records[0]+ assert len(records) == 4+ assert [+ line for line in result.splitlines() if line.startswith("ID -")+ ] == ["ID - ref1", "ID - ref2", "ID - ref003", "ID - ref4"]+ for record, title, suffix, host in zip(+ records,+ ["Before", "Shared", "Shared", "Final"],+ ["before", "shared", "shared", "final"],+ ["example.com", "example.org", "example.org", "example.org"],+ ):+ assert f"TI - {title}\n" in record+ assert f"UR - https://{host}/{suffix}\n" in record+ assert f"DO - 10.1234/{suffix}\n" in record+ for record in records[1:3]:+ assert "AU - Example Author\n" in record+ assert "PY - 2020\n" in record