Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(export,news): malformed RIS export + dropped subscription-history metadata (#4435)

LearningCircuit0dc65b5bb786
diff --git a/changelog.d/4435.bugfix.md b/changelog.d/4435.bugfix.md
new file mode 100644
index 0000000000..d2c1335c9d
--- /dev/null
+++ b/changelog.d/4435.bugfix.md
@@ -0,0 +1 @@
+Two pre-existing export/metadata correctness bugs are fixed. RIS citation exports were malformed: the entry builder reused its output accumulator as a scratch variable when reading the title, so each record emitted the raw source text *before* the mandatory leading ``TY - `` tag and reference managers (Zotero/Mendeley/EndNote) rejected the file. Separately, ``get_subscription_history`` called ``json.loads()`` on ``research_meta`` — a JSON column that SQLAlchemy already deserializes to a dict on read — so the resulting ``TypeError`` was swallowed by a bare ``except`` and every subscription-history item silently lost its headline and topics; it now handles both dict and legacy-string values like the rest of the module.
diff --git a/src/local_deep_research/news/api.py b/src/local_deep_research/news/api.py
index 8ce7db738b..a1cbf21a06 100644
--- a/src/local_deep_research/news/api.py
+++ b/src/local_deep_research/news/api.py
@@ -569,7 +569,19 @@ def get_subscription_history(
# Parse metadata if available to get headline and topics
if item.get("research_meta"):
try:
- meta = json.loads(item["research_meta"])
+ # research_meta is a JSON column, so SQLAlchemy already
+ # deserializes it to a dict on read; only legacy/text rows
+ # arrive as a JSON string. Calling json.loads() on the dict
+ # raised TypeError that the bare except below swallowed,
+ # silently blanking the headline/topics for every item.
+ # Mirror the dict/str handling already used at the top of
+ # this module (see get_news_feed).
+ raw_meta = item["research_meta"]
+ meta = (
+ json.loads(raw_meta)
+ if isinstance(raw_meta, str)
+ else raw_meta
+ )
processed_item["triggered_by"] = meta.get(
"triggered_by", "subscription"
)
diff --git a/src/local_deep_research/text_optimization/citation_formatter.py b/src/local_deep_research/text_optimization/citation_formatter.py
index 16b9a4e0f1..1c939ae69a 100644
--- a/src/local_deep_research/text_optimization/citation_formatter.py
+++ b/src/local_deep_research/text_optimization/citation_formatter.py
@@ -918,9 +918,13 @@ class RISExporter:
if metadata is None:
metadata = {}
- # Extract title from first line
- lines = full_text.split("\n")
- title = lines[0].strip()
+ # Extract title from first line. NB: split into a *separate* variable —
+ # ``lines`` is the RIS-output accumulator initialized above and appended
+ # to below; reusing it here previously overwrote it with the source
+ # text, so every entry emitted the raw source body before the mandatory
+ # leading ``TY - `` tag and reference managers rejected the file.
+ text_lines = full_text.split("\n")
+ title = text_lines[0].strip()
# Extract year from full text (looks for 4-digit year)
year_match = re.search(r"\b(19\d{2}|20\d{2})\b", full_text)
diff --git a/tests/citation_bugs/test_citation_export_formats.py b/tests/citation_bugs/test_citation_export_formats.py
index a17e6fcfad..a5a482aa58 100644
--- a/tests/citation_bugs/test_citation_export_formats.py
+++ b/tests/citation_bugs/test_citation_export_formats.py
@@ -179,8 +179,9 @@ URL: https://same.url/paper"""
ris_exporter = RISExporter()
ris_output = ris_exporter.export_to_ris(content)
assert ris_output.count("TY - ELEC") == 2
- # Each URL appears twice: once in the original source text and once in the RIS entry
- assert ris_output.count("https://same.url/paper") == 4
+ # The fixed exporter emits clean RIS records only (no echo of the
+ # raw source text), so the shared URL appears once per entry → 2.
+ assert ris_output.count("https://same.url/paper") == 2
# LaTeX should have separate bibitems
latex_exporter = LaTeXExporter()
diff --git a/tests/news/test_news_api_extended.py b/tests/news/test_news_api_extended.py
index 3396ccceeb..d54a4712b8 100644
--- a/tests/news/test_news_api_extended.py
+++ b/tests/news/test_news_api_extended.py
@@ -916,6 +916,52 @@ class TestSubscriptionHistory:
mock_get_session.return_value.__exit__ = Mock(return_value=False)
return get_subscription_history("sub_1")
+ def _run_history_with_meta(self, research_meta):
+ """Drive get_subscription_history with one history row whose
+ research_meta is ``research_meta``, returning the first processed
+ history item. Used by the dict/string regression tests below."""
+ from local_deep_research.news.api import get_subscription_history
+
+ subscription = MagicMock()
+ subscription.created_at = None
+ subscription.next_refresh = None
+ subscription.refresh_count = 0
+
+ research = MagicMock()
+ research.id = "res-1"
+ research.query = "quantum computing breakthroughs"
+ research.status = "completed"
+ research.created_at = None
+ research.completed_at = None
+ research.duration_seconds = 12
+ research.report_path = None
+ research.research_meta = research_meta
+
+ mock_session = MagicMock()
+ mock_query = MagicMock()
+ mock_session.query.return_value = mock_query
+ # Subscription lookup: .filter_by(...).first()
+ mock_query.filter_by.return_value = mock_query
+ mock_query.first.return_value = subscription
+ # History lookup: .filter(...).order_by(...).limit(...).all()
+ mock_query.filter.return_value = mock_query
+ mock_query.order_by.return_value = mock_query
+ mock_query.limit.return_value = mock_query
+ mock_query.all.return_value = [research]
+
+ with patch(
+ "local_deep_research.database.session_context.get_user_db_session"
+ ) as mock_get_session:
+ mock_get_session.return_value.__enter__ = Mock(
+ return_value=mock_session
+ )
+ mock_get_session.return_value.__exit__ = Mock(return_value=False)
+
+ result = get_subscription_history("sub-1")
+
+ assert result["history"], "expected one history item"
+ return result["history"][0]
+
def test_real_subscription_has_no_refresh_count_column(self):
"""Regression: NewsSubscription has no refresh_count column. Reading it
raised AttributeError and 500'd the endpoint. With no runs, refresh_count
@@ -965,6 +1011,36 @@ class TestSubscriptionHistory:
assert result["subscription"]["refresh_count"] == 3
assert result["total_runs"] == 3
+ def test_dict_research_meta_populates_headline_and_topics(self):
+ """research_meta is a JSON column, so it deserializes to a dict on
+ read. The old code called json.loads() on that dict, raising a
+ TypeError that the bare except swallowed and blanked the headline and
+ topics for every history item. With a dict it must now populate them.
+ """
+ item = self._run_history_with_meta(
+ {
+ "triggered_by": "subscription",
+ "generated_headline": "Quantum Leap",
+ "generated_topics": ["physics", "computing"],
+ }
+ )
+ assert item["headline"] == "Quantum Leap"
+ assert item["topics"] == ["physics", "computing"]
+ assert item["triggered_by"] == "subscription"
+
+ def test_string_research_meta_still_parsed(self):
+ """Legacy/text rows arriving as a JSON string are still parsed."""
+ item = self._run_history_with_meta(
+ json.dumps(
+ {
+ "generated_headline": "Legacy Headline",
+ "generated_topics": ["history"],
+ }
+ )
+ )
+ assert item["headline"] == "Legacy Headline"
+ assert item["topics"] == ["history"]
+
class TestVoteFunctions:
"""Tests for vote/feedback functions."""
diff --git a/tests/text_optimization/test_ris_entry_edge_cases.py b/tests/text_optimization/test_ris_entry_edge_cases.py
index f26318afd5..a00b90a558 100644
--- a/tests/text_optimization/test_ris_entry_edge_cases.py
+++ b/tests/text_optimization/test_ris_entry_edge_cases.py
@@ -14,6 +14,8 @@ This file adds coverage for the many remaining branches:
- Author match overlapping with title cleaning
"""
+import re
+
from freezegun import freeze_time
from local_deep_research.text_optimization.citation_formatter import RISExporter
@@ -439,3 +441,50 @@ class TestRISEntryURLHandling:
url = "https://example.com/path/to/article?id=123"
result = exporter._create_ris_entry("1", "Title", url=url)
assert f"UR - {url}" in result
+
+
+class TestRISEntryStructuralValidity:
+ """Regression guard for the accumulator-clobber bug.
+
+ ``_create_ris_entry`` initialized ``lines = []`` as the RIS-output
+ accumulator, then immediately reassigned ``lines = full_text.split("\\n")``
+ to read the title — overwriting the accumulator with the raw source body.
+ Every RIS field was then appended *after* those source lines, so each entry
+ emitted the source text before the mandatory leading ``TY - `` tag and
+ reference managers (Zotero/Mendeley/EndNote) rejected the record. The
+ pre-existing tests only used ``"<tag>" in result`` substring checks, so they
+ never caught the leaked prefix.
+ """
+
+ # RIS records are newline-separated ``XX - value`` lines where the tag is
+ # two uppercase-or-digit chars (TY, ID, TI, AU, DO, PY, UR, PB, Y1, DA, LA,
+ # ER), two spaces, a hyphen, then a space.
+ _RIS_LINE = re.compile(r"^[A-Z][A-Z0-9] - ")
+
+ def test_entry_starts_with_ty_tag(self):
+ """RIS requires ``TY`` to be the first tag of the record."""
+ exporter = RISExporter()
+ source = (
+ "Understanding Widgets\n"
+ "URL: https://example.com/widgets\n"
+ "Collection: Gadgets"
+ )
+ result = exporter._create_ris_entry(
+ "7", source, url="https://example.com/widgets"
+ )
+ assert result.startswith("TY - "), result
+
+ def test_no_raw_source_lines_leak(self):
+ """Every non-blank output line is a RIS tag, never raw source body."""
+ exporter = RISExporter()
+ source = (
+ "My Title\n"
+ "BODYLEAKMARKER this descriptive line must never be emitted\n"
+ "Collection: Internal"
+ )
+ result = exporter._create_ris_entry("1", source)
+ assert "BODYLEAKMARKER" not in result
+ for line in result.split("\n"):
+ if not line.strip():
+ continue
+ assert self._RIS_LINE.match(line), f"non-RIS line leaked: {line!r}"