Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(metrics): project link-analytics ResearchResource scan to bound memory (#4560) (#4728)

LearningCircuit15ef6c2d6245
diff --git a/changelog.d/+bound-link-analytics-scan.bugfix.md b/changelog.d/+bound-link-analytics-scan.bugfix.md
new file mode 100644
index 0000000000..4d50540c4d
--- /dev/null
+++ b/changelog.d/+bound-link-analytics-scan.bugfix.md
@@ -0,0 +1 @@
+Link analytics (`GET /api/link-analytics`) now projects only the columns it needs plus a SQL-level "has preview" flag, instead of loading full research-resource rows. This avoids materializing every resource's `content_preview` text on the whole-table scan that runs when no time filter is applied (#4560).
diff --git a/src/local_deep_research/web/routes/metrics_routes.py b/src/local_deep_research/web/routes/metrics_routes.py
index 4cd32a486c..751f3ed33c 100644
--- a/src/local_deep_research/web/routes/metrics_routes.py
+++ b/src/local_deep_research/web/routes/metrics_routes.py
@@ -178,8 +178,24 @@ def get_link_analytics(period="30d", username=None):
days = get_period_days(period)
with get_user_db_session(username) as session:
- # Base query
- query = session.query(ResearchResource)
+ # Project only the columns the analytics loop reads (url /
+ # research_id / created_at / source_type / title) plus a SQL-level
+ # boolean for whether content_preview is present. Loading full
+ # ``ResearchResource`` entities would materialize the
+ # ``content_preview`` Text column for every row on this whole-table
+ # scan — which runs unfiltered when ``period=all`` (#4560).
+ has_preview = (
+ ResearchResource.content_preview.isnot(None)
+ & (ResearchResource.content_preview != "")
+ ).label("has_preview")
+ query = session.query(
+ ResearchResource.url,
+ ResearchResource.research_id,
+ ResearchResource.created_at,
+ ResearchResource.source_type,
+ ResearchResource.title,
+ has_preview,
+ )
# Apply time filter
if days:
@@ -293,9 +309,9 @@ def get_link_analytics(period="30d", username=None):
quality_metrics["total"] += 1
if resource.title:
quality_metrics["with_title"] += 1
- if resource.content_preview:
+ if resource.has_preview:
quality_metrics["with_preview"] += 1
- if resource.title and resource.content_preview:
+ if resource.title and resource.has_preview:
quality_metrics["with_both"] += 1
# Track domain co-occurrences for network visualization
diff --git a/tests/test_link_analytics.py b/tests/test_link_analytics.py
index d156750e51..9de2b02b53 100644
--- a/tests/test_link_analytics.py
+++ b/tests/test_link_analytics.py
@@ -73,6 +73,12 @@ def mock_resources():
).isoformat()
resource.title = f"Resource {i}"
resource.content_preview = f"Content preview for resource {i}"
+ # The route projects a SQL-level ``has_preview`` boolean rather than
+ # the full content_preview body; mirror that here so the spec'd mock
+ # exposes the attribute the loop actually reads (otherwise the access
+ # raises AttributeError, is swallowed by the per-row except, and
+ # quality_metrics / domain_connections silently never populate).
+ resource.has_preview = bool(resource.content_preview)
resources.append(resource)
return resources
@@ -107,6 +113,70 @@ class TestLinkAnalytics:
assert analytics["avg_links_per_research"] == 0
assert len(analytics["top_domains"]) == 0
+ def test_get_link_analytics_real_db_projection(self):
+ """Real-DB guard (#4560): exercise get_link_analytics against a real
+ in-memory DB so the projected query (url/research_id/created_at/
+ source_type/title + a ``has_preview`` SQL boolean — NOT the full
+ ``content_preview`` body) runs on real ``Row`` objects.
+
+ A future consumer reading a non-projected column would AttributeError
+ in the per-row loop (swallowed by its broad ``except``) and silently
+ degrade the analytics output — which the attribute-permissive mock
+ tests cannot catch. These output assertions would fail in that case.
+ """
+ from contextlib import contextmanager
+
+ from sqlalchemy import create_engine
+ from sqlalchemy.orm import Session as SASession
+
+ from local_deep_research.database.models import Base, ResearchResource
+
+ engine = create_engine("sqlite://")
+ Base.metadata.create_all(engine)
+ with SASession(engine) as setup:
+ setup.add_all(
+ [
+ ResearchResource(
+ research_id="r1",
+ url="https://arxiv.org/abs/1",
+ title="A",
+ content_preview="body",
+ source_type="web",
+ created_at="2026-06-10T00:00:00",
+ ),
+ ResearchResource(
+ research_id="r1",
+ url="https://nature.com/x",
+ title=None,
+ content_preview="",
+ source_type=None,
+ created_at="2026-06-11T00:00:00",
+ ),
+ ]
+ )
+ setup.commit()
+
+ @contextmanager
+ def _real_session(username, *args, **kwargs):
+ with SASession(engine) as s:
+ yield s
+
+ with patch(
+ "local_deep_research.web.routes.metrics_routes.get_user_db_session",
+ _real_session,
+ ):
+ result = get_link_analytics(period="all", username="test_user")
+
+ la = result["link_analytics"]
+ # The projected query ran end-to-end on real Rows with no swallowed
+ # AttributeError degrading the output:
+ assert la["total_links"] == 2 # query returned both rows
+ assert la["total_unique_domains"] == 2 # url projected + parsed
+ assert set(la["source_type_analysis"]) == {
+ "web"
+ } # source_type projected
+ assert la["total_researches"] == 1 # research_id projected (both "r1")
+
def test_get_link_analytics_with_data(self, mock_resources):
"""Test analytics with mock resources."""
with patch(
@@ -188,6 +258,7 @@ class TestLinkAnalytics:
source_type=None,
title="Example",
content_preview=None,
+ has_preview=False,
created_at="2024-01-01T00:00:00Z",
),
MagicMock(
@@ -196,6 +267,7 @@ class TestLinkAnalytics:
source_type=None,
title="Python Docs",
content_preview=None,
+ has_preview=False,
created_at="2024-01-01T00:00:00Z",
),
MagicMock(
@@ -204,6 +276,7 @@ class TestLinkAnalytics:
source_type=None,
title="GitHub",
content_preview=None,
+ has_preview=False,
created_at="2024-01-01T00:00:00Z",
),
MagicMock(
@@ -212,6 +285,7 @@ class TestLinkAnalytics:
source_type=None,
title="GitHub 2",
content_preview=None,
+ has_preview=False,
created_at="2024-01-01T00:00:00Z",
),
]
diff --git a/tests/web/routes/test_metrics_analytics_logic.py b/tests/web/routes/test_metrics_analytics_logic.py
index 30f9ad2d85..5cc9c9330b 100644
--- a/tests/web/routes/test_metrics_analytics_logic.py
+++ b/tests/web/routes/test_metrics_analytics_logic.py
@@ -7,7 +7,6 @@ from unittest.mock import MagicMock, Mock, patch
from local_deep_research.database.models import (
ResearchHistory,
- ResearchResource,
)
from local_deep_research.domain_classifier import DomainClassification
from local_deep_research.web.routes.metrics_routes import (
@@ -229,15 +228,16 @@ def _build_link_session(resources, classifications=None, researches=None):
session = MagicMock()
- def query_dispatch(model):
- if model is ResearchResource:
- return resource_query
- if model is DomainClassification:
+ def query_dispatch(*entities):
+ # ResearchResource is now queried by projected columns (id/url/... +
+ # a has_preview label) rather than the full entity (#4560), so match
+ # the other two models by identity and default to the resource query.
+ first = entities[0]
+ if first is DomainClassification:
return classification_query
- if model is ResearchHistory:
+ if first is ResearchHistory:
return research_query
- # Fallback for any other model
- return MagicMock()
+ return resource_query
session.query.side_effect = query_dispatch
return session
@@ -256,6 +256,9 @@ def _make_resource(
r.research_id = research_id
r.title = title
r.content_preview = preview
+ # get_link_analytics now reads the SQL-projected ``has_preview`` boolean
+ # (content_preview IS NOT NULL AND != '') rather than the body itself (#4560).
+ r.has_preview = bool(preview)
r.created_at = created_at
r.source_type = source_type
return r