~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(metrics): project link-analytics ResearchResource scan to bound memory (#4560) (#4728)
LearningCircuit15ef6c2d6245
diff --git a/changelog.d/+bound-link-analytics-scan.bugfix.md b/changelog.d/+bound-link-analytics-scan.bugfix.mdnew file mode 100644index 0000000000..4d50540c4d--- /dev/null+++ b/changelog.d/+bound-link-analytics-scan.bugfix.md@@ -0,0 +1 @@+Link analytics (`GET /api/link-analytics`) now projects only the columns it needs plus a SQL-level "has preview" flag, instead of loading full research-resource rows. This avoids materializing every resource's `content_preview` text on the whole-table scan that runs when no time filter is applied (#4560).diff --git a/src/local_deep_research/web/routes/metrics_routes.py b/src/local_deep_research/web/routes/metrics_routes.pyindex 4cd32a486c..751f3ed33c 100644--- a/src/local_deep_research/web/routes/metrics_routes.py+++ b/src/local_deep_research/web/routes/metrics_routes.py@@ -178,8 +178,24 @@ def get_link_analytics(period="30d", username=None):days = get_period_days(period)with get_user_db_session(username) as session:- # Base query- query = session.query(ResearchResource)+ # Project only the columns the analytics loop reads (url /+ # research_id / created_at / source_type / title) plus a SQL-level+ # boolean for whether content_preview is present. Loading full+ # ``ResearchResource`` entities would materialize the+ # ``content_preview`` Text column for every row on this whole-table+ # scan — which runs unfiltered when ``period=all`` (#4560).+ has_preview = (+ ResearchResource.content_preview.isnot(None)+ & (ResearchResource.content_preview != "")+ ).label("has_preview")+ query = session.query(+ ResearchResource.url,+ ResearchResource.research_id,+ ResearchResource.created_at,+ ResearchResource.source_type,+ ResearchResource.title,+ has_preview,+ )# Apply time filterif days:@@ -293,9 +309,9 @@ def get_link_analytics(period="30d", username=None):quality_metrics["total"] += 1if resource.title:quality_metrics["with_title"] += 1- if resource.content_preview:+ if resource.has_preview:quality_metrics["with_preview"] += 1- if resource.title and resource.content_preview:+ if resource.title and resource.has_preview:quality_metrics["with_both"] += 1# Track domain co-occurrences for network visualizationdiff --git a/tests/test_link_analytics.py b/tests/test_link_analytics.pyindex d156750e51..9de2b02b53 100644--- a/tests/test_link_analytics.py+++ b/tests/test_link_analytics.py@@ -73,6 +73,12 @@ def mock_resources():).isoformat()resource.title = f"Resource {i}"resource.content_preview = f"Content preview for resource {i}"+ # The route projects a SQL-level ``has_preview`` boolean rather than+ # the full content_preview body; mirror that here so the spec'd mock+ # exposes the attribute the loop actually reads (otherwise the access+ # raises AttributeError, is swallowed by the per-row except, and+ # quality_metrics / domain_connections silently never populate).+ resource.has_preview = bool(resource.content_preview)resources.append(resource)return resources@@ -107,6 +113,70 @@ class TestLinkAnalytics:assert analytics["avg_links_per_research"] == 0assert len(analytics["top_domains"]) == 0+ def test_get_link_analytics_real_db_projection(self):+ """Real-DB guard (#4560): exercise get_link_analytics against a real+ in-memory DB so the projected query (url/research_id/created_at/+ source_type/title + a ``has_preview`` SQL boolean — NOT the full+ ``content_preview`` body) runs on real ``Row`` objects.++ A future consumer reading a non-projected column would AttributeError+ in the per-row loop (swallowed by its broad ``except``) and silently+ degrade the analytics output — which the attribute-permissive mock+ tests cannot catch. These output assertions would fail in that case.+ """+ from contextlib import contextmanager++ from sqlalchemy import create_engine+ from sqlalchemy.orm import Session as SASession++ from local_deep_research.database.models import Base, ResearchResource++ engine = create_engine("sqlite://")+ Base.metadata.create_all(engine)+ with SASession(engine) as setup:+ setup.add_all(+ [+ ResearchResource(+ research_id="r1",+ url="https://arxiv.org/abs/1",+ title="A",+ content_preview="body",+ source_type="web",+ created_at="2026-06-10T00:00:00",+ ),+ ResearchResource(+ research_id="r1",+ url="https://nature.com/x",+ title=None,+ content_preview="",+ source_type=None,+ created_at="2026-06-11T00:00:00",+ ),+ ]+ )+ setup.commit()++ @contextmanager+ def _real_session(username, *args, **kwargs):+ with SASession(engine) as s:+ yield s++ with patch(+ "local_deep_research.web.routes.metrics_routes.get_user_db_session",+ _real_session,+ ):+ result = get_link_analytics(period="all", username="test_user")++ la = result["link_analytics"]+ # The projected query ran end-to-end on real Rows with no swallowed+ # AttributeError degrading the output:+ assert la["total_links"] == 2 # query returned both rows+ assert la["total_unique_domains"] == 2 # url projected + parsed+ assert set(la["source_type_analysis"]) == {+ "web"+ } # source_type projected+ assert la["total_researches"] == 1 # research_id projected (both "r1")+def test_get_link_analytics_with_data(self, mock_resources):"""Test analytics with mock resources."""with patch(@@ -188,6 +258,7 @@ class TestLinkAnalytics:source_type=None,title="Example",content_preview=None,+ has_preview=False,created_at="2024-01-01T00:00:00Z",),MagicMock(@@ -196,6 +267,7 @@ class TestLinkAnalytics:source_type=None,title="Python Docs",content_preview=None,+ has_preview=False,created_at="2024-01-01T00:00:00Z",),MagicMock(@@ -204,6 +276,7 @@ class TestLinkAnalytics:source_type=None,title="GitHub",content_preview=None,+ has_preview=False,created_at="2024-01-01T00:00:00Z",),MagicMock(@@ -212,6 +285,7 @@ class TestLinkAnalytics:source_type=None,title="GitHub 2",content_preview=None,+ has_preview=False,created_at="2024-01-01T00:00:00Z",),]diff --git a/tests/web/routes/test_metrics_analytics_logic.py b/tests/web/routes/test_metrics_analytics_logic.pyindex 30f9ad2d85..5cc9c9330b 100644--- a/tests/web/routes/test_metrics_analytics_logic.py+++ b/tests/web/routes/test_metrics_analytics_logic.py@@ -7,7 +7,6 @@ from unittest.mock import MagicMock, Mock, patchfrom local_deep_research.database.models import (ResearchHistory,- ResearchResource,)from local_deep_research.domain_classifier import DomainClassificationfrom local_deep_research.web.routes.metrics_routes import (@@ -229,15 +228,16 @@ def _build_link_session(resources, classifications=None, researches=None):session = MagicMock()- def query_dispatch(model):- if model is ResearchResource:- return resource_query- if model is DomainClassification:+ def query_dispatch(*entities):+ # ResearchResource is now queried by projected columns (id/url/... ++ # a has_preview label) rather than the full entity (#4560), so match+ # the other two models by identity and default to the resource query.+ first = entities[0]+ if first is DomainClassification:return classification_query- if model is ResearchHistory:+ if first is ResearchHistory:return research_query- # Fallback for any other model- return MagicMock()+ return resource_querysession.query.side_effect = query_dispatchreturn session@@ -256,6 +256,9 @@ def _make_resource(r.research_id = research_idr.title = titler.content_preview = preview+ # get_link_analytics now reads the SQL-projected ``has_preview`` boolean+ # (content_preview IS NOT NULL AND != '') rather than the body itself (#4560).+ r.has_preview = bool(preview)r.created_at = created_atr.source_type = source_typereturn r