~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(export,news): malformed RIS export + dropped subscription-history metadata (#4435)
LearningCircuit0dc65b5bb786
diff --git a/changelog.d/4435.bugfix.md b/changelog.d/4435.bugfix.mdnew file mode 100644index 0000000000..d2c1335c9d--- /dev/null+++ b/changelog.d/4435.bugfix.md@@ -0,0 +1 @@+Two pre-existing export/metadata correctness bugs are fixed. RIS citation exports were malformed: the entry builder reused its output accumulator as a scratch variable when reading the title, so each record emitted the raw source text *before* the mandatory leading ``TY - `` tag and reference managers (Zotero/Mendeley/EndNote) rejected the file. Separately, ``get_subscription_history`` called ``json.loads()`` on ``research_meta`` — a JSON column that SQLAlchemy already deserializes to a dict on read — so the resulting ``TypeError`` was swallowed by a bare ``except`` and every subscription-history item silently lost its headline and topics; it now handles both dict and legacy-string values like the rest of the module.diff --git a/src/local_deep_research/news/api.py b/src/local_deep_research/news/api.pyindex 8ce7db738b..a1cbf21a06 100644--- a/src/local_deep_research/news/api.py+++ b/src/local_deep_research/news/api.py@@ -569,7 +569,19 @@ def get_subscription_history(# Parse metadata if available to get headline and topicsif item.get("research_meta"):try:- meta = json.loads(item["research_meta"])+ # research_meta is a JSON column, so SQLAlchemy already+ # deserializes it to a dict on read; only legacy/text rows+ # arrive as a JSON string. Calling json.loads() on the dict+ # raised TypeError that the bare except below swallowed,+ # silently blanking the headline/topics for every item.+ # Mirror the dict/str handling already used at the top of+ # this module (see get_news_feed).+ raw_meta = item["research_meta"]+ meta = (+ json.loads(raw_meta)+ if isinstance(raw_meta, str)+ else raw_meta+ )processed_item["triggered_by"] = meta.get("triggered_by", "subscription")diff --git a/src/local_deep_research/text_optimization/citation_formatter.py b/src/local_deep_research/text_optimization/citation_formatter.pyindex 16b9a4e0f1..1c939ae69a 100644--- a/src/local_deep_research/text_optimization/citation_formatter.py+++ b/src/local_deep_research/text_optimization/citation_formatter.py@@ -918,9 +918,13 @@ class RISExporter:if metadata is None:metadata = {}- # Extract title from first line- lines = full_text.split("\n")- title = lines[0].strip()+ # Extract title from first line. NB: split into a *separate* variable —+ # ``lines`` is the RIS-output accumulator initialized above and appended+ # to below; reusing it here previously overwrote it with the source+ # text, so every entry emitted the raw source body before the mandatory+ # leading ``TY - `` tag and reference managers rejected the file.+ text_lines = full_text.split("\n")+ title = text_lines[0].strip()# Extract year from full text (looks for 4-digit year)year_match = re.search(r"\b(19\d{2}|20\d{2})\b", full_text)diff --git a/tests/citation_bugs/test_citation_export_formats.py b/tests/citation_bugs/test_citation_export_formats.pyindex a17e6fcfad..a5a482aa58 100644--- a/tests/citation_bugs/test_citation_export_formats.py+++ b/tests/citation_bugs/test_citation_export_formats.py@@ -179,8 +179,9 @@ URL: https://same.url/paper"""ris_exporter = RISExporter()ris_output = ris_exporter.export_to_ris(content)assert ris_output.count("TY - ELEC") == 2- # Each URL appears twice: once in the original source text and once in the RIS entry- assert ris_output.count("https://same.url/paper") == 4+ # The fixed exporter emits clean RIS records only (no echo of the+ # raw source text), so the shared URL appears once per entry → 2.+ assert ris_output.count("https://same.url/paper") == 2# LaTeX should have separate bibitemslatex_exporter = LaTeXExporter()diff --git a/tests/news/test_news_api_extended.py b/tests/news/test_news_api_extended.pyindex 3396ccceeb..d54a4712b8 100644--- a/tests/news/test_news_api_extended.py+++ b/tests/news/test_news_api_extended.py@@ -916,6 +916,52 @@ class TestSubscriptionHistory:mock_get_session.return_value.__exit__ = Mock(return_value=False)return get_subscription_history("sub_1")+ def _run_history_with_meta(self, research_meta):+ """Drive get_subscription_history with one history row whose+ research_meta is ``research_meta``, returning the first processed+ history item. Used by the dict/string regression tests below."""+ from local_deep_research.news.api import get_subscription_history++ subscription = MagicMock()+ subscription.created_at = None+ subscription.next_refresh = None+ subscription.refresh_count = 0++ research = MagicMock()+ research.id = "res-1"+ research.query = "quantum computing breakthroughs"+ research.status = "completed"+ research.created_at = None+ research.completed_at = None+ research.duration_seconds = 12+ research.report_path = None+ research.research_meta = research_meta++ mock_session = MagicMock()+ mock_query = MagicMock()+ mock_session.query.return_value = mock_query+ # Subscription lookup: .filter_by(...).first()+ mock_query.filter_by.return_value = mock_query+ mock_query.first.return_value = subscription+ # History lookup: .filter(...).order_by(...).limit(...).all()+ mock_query.filter.return_value = mock_query+ mock_query.order_by.return_value = mock_query+ mock_query.limit.return_value = mock_query+ mock_query.all.return_value = [research]++ with patch(+ "local_deep_research.database.session_context.get_user_db_session"+ ) as mock_get_session:+ mock_get_session.return_value.__enter__ = Mock(+ return_value=mock_session+ )+ mock_get_session.return_value.__exit__ = Mock(return_value=False)++ result = get_subscription_history("sub-1")++ assert result["history"], "expected one history item"+ return result["history"][0]+def test_real_subscription_has_no_refresh_count_column(self):"""Regression: NewsSubscription has no refresh_count column. Reading itraised AttributeError and 500'd the endpoint. With no runs, refresh_count@@ -965,6 +1011,36 @@ class TestSubscriptionHistory:assert result["subscription"]["refresh_count"] == 3assert result["total_runs"] == 3+ def test_dict_research_meta_populates_headline_and_topics(self):+ """research_meta is a JSON column, so it deserializes to a dict on+ read. The old code called json.loads() on that dict, raising a+ TypeError that the bare except swallowed and blanked the headline and+ topics for every history item. With a dict it must now populate them.+ """+ item = self._run_history_with_meta(+ {+ "triggered_by": "subscription",+ "generated_headline": "Quantum Leap",+ "generated_topics": ["physics", "computing"],+ }+ )+ assert item["headline"] == "Quantum Leap"+ assert item["topics"] == ["physics", "computing"]+ assert item["triggered_by"] == "subscription"++ def test_string_research_meta_still_parsed(self):+ """Legacy/text rows arriving as a JSON string are still parsed."""+ item = self._run_history_with_meta(+ json.dumps(+ {+ "generated_headline": "Legacy Headline",+ "generated_topics": ["history"],+ }+ )+ )+ assert item["headline"] == "Legacy Headline"+ assert item["topics"] == ["history"]+class TestVoteFunctions:"""Tests for vote/feedback functions."""diff --git a/tests/text_optimization/test_ris_entry_edge_cases.py b/tests/text_optimization/test_ris_entry_edge_cases.pyindex f26318afd5..a00b90a558 100644--- a/tests/text_optimization/test_ris_entry_edge_cases.py+++ b/tests/text_optimization/test_ris_entry_edge_cases.py@@ -14,6 +14,8 @@ This file adds coverage for the many remaining branches:- Author match overlapping with title cleaning"""+import re+from freezegun import freeze_timefrom local_deep_research.text_optimization.citation_formatter import RISExporter@@ -439,3 +441,50 @@ class TestRISEntryURLHandling:url = "https://example.com/path/to/article?id=123"result = exporter._create_ris_entry("1", "Title", url=url)assert f"UR - {url}" in result+++class TestRISEntryStructuralValidity:+ """Regression guard for the accumulator-clobber bug.++ ``_create_ris_entry`` initialized ``lines = []`` as the RIS-output+ accumulator, then immediately reassigned ``lines = full_text.split("\\n")``+ to read the title — overwriting the accumulator with the raw source body.+ Every RIS field was then appended *after* those source lines, so each entry+ emitted the source text before the mandatory leading ``TY - `` tag and+ reference managers (Zotero/Mendeley/EndNote) rejected the record. The+ pre-existing tests only used ``"<tag>" in result`` substring checks, so they+ never caught the leaked prefix.+ """++ # RIS records are newline-separated ``XX - value`` lines where the tag is+ # two uppercase-or-digit chars (TY, ID, TI, AU, DO, PY, UR, PB, Y1, DA, LA,+ # ER), two spaces, a hyphen, then a space.+ _RIS_LINE = re.compile(r"^[A-Z][A-Z0-9] - ")++ def test_entry_starts_with_ty_tag(self):+ """RIS requires ``TY`` to be the first tag of the record."""+ exporter = RISExporter()+ source = (+ "Understanding Widgets\n"+ "URL: https://example.com/widgets\n"+ "Collection: Gadgets"+ )+ result = exporter._create_ris_entry(+ "7", source, url="https://example.com/widgets"+ )+ assert result.startswith("TY - "), result++ def test_no_raw_source_lines_leak(self):+ """Every non-blank output line is a RIS tag, never raw source body."""+ exporter = RISExporter()+ source = (+ "My Title\n"+ "BODYLEAKMARKER this descriptive line must never be emitted\n"+ "Collection: Internal"+ )+ result = exporter._create_ris_entry("1", source)+ assert "BODYLEAKMARKER" not in result+ for line in result.split("\n"):+ if not line.strip():+ continue+ assert self._RIS_LINE.match(line), f"non-RIS line leaked: {line!r}"