~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(memory): cap persisted log message size + share /history/logs cap constants (#4240)
LearningCircuitc96449482f3f
diff --git a/changelog.d/+log-message-db-cap.bugfix.md b/changelog.d/+log-message-db-cap.bugfix.mdnew file mode 100644index 0000000000..3b89f8d15b--- /dev/null+++ b/changelog.d/+log-message-db-cap.bugfix.md@@ -0,0 +1 @@+Cap persisted log messages at 5000 chars so long langgraph runs no longer accumulate 10 KB ``ResearchLog`` rows. Mirrors PR #4004's frontend truncation discipline, now applied to the database sink. Also promotes the duplicated ``/history/logs`` pagination-cap literals into a single shared constant (``HISTORY_LOGS_HARD_CAP`` / ``HISTORY_LOGS_DEFAULT_LIMIT``, exposed to the log panel as ``window.LDR_LOG_LIMITS``) so the route clamp and the panel's DOM cap no longer drift. Full diagnostics remain available in stderr/file sinks.diff --git a/src/local_deep_research/constants.py b/src/local_deep_research/constants.pyindex 253760a31f..85ca490e6f 100644--- a/src/local_deep_research/constants.py+++ b/src/local_deep_research/constants.py@@ -97,6 +97,14 @@ DEFAULT_LOCAL_SEARCH_TEXT_SEPARATORS_JSON = '["\\n\\n", "\\n", ". ", " ", ""]'SNIPPET_LENGTH_SHORT = 250SNIPPET_LENGTH_LONG = 500+# --- /history/logs/<id> pagination caps ---+# Default matches the frontend logpanel DOM cap (MAX_LOG_ENTRIES); the+# hard cap is the ceiling the route clamps to so a client cannot force+# an unbounded load. Shared with the frontend via inject_frontend_constants+# (see web/app_factory.py) → window.LDR_LOG_LIMITS (see base.html).+HISTORY_LOGS_DEFAULT_LIMIT = 500+HISTORY_LOGS_HARD_CAP = 5000+# --- Research history collection ---RESEARCH_HISTORY_COLLECTION_NAME = "History"RESEARCH_HISTORY_COLLECTION_DESCRIPTION = (diff --git a/src/local_deep_research/utilities/log_utils.py b/src/local_deep_research/utilities/log_utils.pyindex 5d22e85b60..2e9393392d 100644--- a/src/local_deep_research/utilities/log_utils.py+++ b/src/local_deep_research/utilities/log_utils.py@@ -47,6 +47,14 @@ Default log directory to use.# bytes), so it can exceed this value by that fixed overhead.FRONTEND_MESSAGE_MAX_LENGTH = 2000+# Cap the size of messages persisted to ResearchLog. The DB column is+# unbounded TEXT, so a long langgraph run can accumulate thousands of+# 10 KB rows — paginated reads (PR #4037) hide the symptom but don't+# stop the storage growth. Same prefix-plus-indicator semantics as+# FRONTEND_MESSAGE_MAX_LENGTH; container-log/stderr/file sinks remain+# unchanged so full diagnostics are still preserved out-of-band.+DATABASE_MESSAGE_MAX_LENGTH = 5000+class InterceptHandler(logging.Handler):"""@@ -353,7 +361,7 @@ def database_sink(message: loguru.Message) -> None:# Create log entry dictlog_entry = {"timestamp": record["time"],- "message": record["message"],+ "message": _truncate_for_database(record["message"]),"module": record["name"],"function": record["function"],"line_no": int(record["line"]),@@ -377,6 +385,23 @@ def database_sink(message: loguru.Message) -> None:_write_log_to_database(log_entry)+def _truncate_for_database(message: str) -> str:+ """Bound the persisted size of a log message.++ ``DATABASE_MESSAGE_MAX_LENGTH`` is the preserved-prefix length;+ the stored string is at most that plus a ~100-char suffix that+ reports the original length so debug context is not lost. Full+ messages remain available in container-log/stderr/file sinks.+ """+ if len(message) <= DATABASE_MESSAGE_MAX_LENGTH:+ return message+ suffix = (+ f"… (truncated; full message in server logs; "+ f"original length: {len(message)} chars)"+ )+ return message[:DATABASE_MESSAGE_MAX_LENGTH] + suffix++def _truncate_for_frontend(message: str) -> str:"""Bound the wire size of an outbound log message.diff --git a/src/local_deep_research/web/app_factory.py b/src/local_deep_research/web/app_factory.pyindex bb4ca57624..8388b4930e 100644--- a/src/local_deep_research/web/app_factory.py+++ b/src/local_deep_research/web/app_factory.py@@ -449,7 +449,11 @@ def apply_middleware(app):# Source of truth: src/local_deep_research/constants.py::ResearchStatus# Frontend helpers: src/local_deep_research/web/static/js/config/constants.js# Template injection: src/local_deep_research/web/templates/base.html- from ..constants import ResearchStatus+ from ..constants import (+ HISTORY_LOGS_DEFAULT_LIMIT,+ HISTORY_LOGS_HARD_CAP,+ ResearchStatus,+ )@app.context_processordef inject_frontend_constants():@@ -491,6 +495,10 @@ def apply_middleware(app):return {"research_status_enum": {m.name: m.value for m in ResearchStatus},"research_terminal_states": [str(s) for s in terminal],+ "log_limits": {+ "default": HISTORY_LOGS_DEFAULT_LIMIT,+ "hard_cap": HISTORY_LOGS_HARD_CAP,+ },"egress_scope": scope,}diff --git a/src/local_deep_research/web/routes/history_routes.py b/src/local_deep_research/web/routes/history_routes.pyindex 8f1c47dfcb..94665e7d42 100644--- a/src/local_deep_research/web/routes/history_routes.py+++ b/src/local_deep_research/web/routes/history_routes.py@@ -4,7 +4,11 @@ from flask import Blueprint, jsonify, request, sessionfrom loguru import loggerfrom sqlalchemy import func-from ...constants import ResearchStatus+from ...constants import (+ HISTORY_LOGS_DEFAULT_LIMIT,+ HISTORY_LOGS_HARD_CAP,+ ResearchStatus,+)from ...database.models import ResearchHistoryfrom ...database.models.library import Document as Documentfrom ...database.session_context import get_user_db_session@@ -399,19 +403,19 @@ def get_markdown(research_id):def get_research_logs(research_id):"""Get logs for a specific research ID.- Accepts ``?limit=N`` to bound the response size; default 500 matches- the frontend's ``MAX_LOG_ENTRIES`` DOM cap. Clamped to ``[1, 5000]``- so a client cannot force an unbounded load (a long langgraph run can- persist thousands of 10 KB rows; pre-cap the route allocated ~150 MB- transient on the server and Firefox parsed a ~50 MB JSON response).+ Accepts ``?limit=N`` to bound the response size; the default matches+ the frontend's ``MAX_LOG_ENTRIES`` DOM cap. Clamped to+ ``[1, HISTORY_LOGS_HARD_CAP]`` so a client cannot force an unbounded+ load (a long langgraph run can persist thousands of rows; pre-cap+ the route allocated ~150 MB transient on the server and Firefox+ parsed a ~50 MB JSON response)."""username = session["username"]- # Per-request cap. 500 matches MAX_LOG_ENTRIES in logpanel.js; the- # 5000 ceiling lets explicit log-download flows still get more rows- # but stops accidental unbounded loads.- limit = request.args.get("limit", default=500, type=int)- limit = max(1, min(limit, 5000))+ limit = request.args.get(+ "limit", default=HISTORY_LOGS_DEFAULT_LIMIT, type=int+ )+ limit = max(1, min(limit, HISTORY_LOGS_HARD_CAP))# First check if the research existswith get_user_db_session(username) as db_session:diff --git a/src/local_deep_research/web/static/js/components/logpanel.js b/src/local_deep_research/web/static/js/components/logpanel.jsindex e63856b8c3..6b78d80088 100644--- a/src/local_deep_research/web/static/js/components/logpanel.js+++ b/src/local_deep_research/web/static/js/components/logpanel.js@@ -17,8 +17,13 @@normalizeTimestamps,} = window.LdrLogHelpers;- // Maximum number of log entries to keep in DOM to prevent unbounded growth- const MAX_LOG_ENTRIES = 500;+ // Maximum number of log entries to keep in DOM to prevent unbounded growth.+ // Seeded from window.LDR_LOG_LIMITS (set in base.html from Python's+ // HISTORY_LOGS_DEFAULT_LIMIT in constants.py) so this DOM cap and the+ // shared pagination default come from one source instead of drifting.+ // Falls back to 500 if the injection is missing (e.g. in a unit-test+ // harness without templates).+ const MAX_LOG_ENTRIES = window.LDR_LOG_LIMITS?.default ?? 500;// Shared state for log panelwindow._logPanelState ||= {@@ -1088,9 +1093,13 @@SafeLogger.warn('downloadLogs called without researchId; skipping');return;}- // Download path requests the server-side hard cap so users still- // get the full tail (server enforces a 5000-row clamp).- fetchLogsForResearch(researchId, 5000).then((logData) => {+ // Download path requests the shared hard cap (window.LDR_LOG_LIMITS,+ // Python's HISTORY_LOGS_HARD_CAP) so users get the full tail. NOTE:+ // the route this fetch hits (/api/research/<id>/logs) does not yet+ // honor ?limit server-side, so this value is currently advisory; the+ // /history/logs route clamps to the same ceiling when used.+ const hardCap = window.LDR_LOG_LIMITS?.hard_cap ?? 5000;+ fetchLogsForResearch(researchId, hardCap).then((logData) => {// Create a blob with the logs dataconst blob = new Blob([JSON.stringify(logData, null, 2)], { type: 'application/json' });diff --git a/src/local_deep_research/web/templates/base.html b/src/local_deep_research/web/templates/base.htmlindex 7ccdb92ad8..7deccdebc9 100644--- a/src/local_deep_research/web/templates/base.html+++ b/src/local_deep_research/web/templates/base.html@@ -218,9 +218,11 @@<!-- Source: src/local_deep_research/constants.py::ResearchStatus --><!-- Injected by: src/local_deep_research/web/app_factory.py (inject_frontend_constants) --><!-- Consumed by: src/local_deep_research/web/static/js/config/constants.js (ResearchStates helpers) -->+ <!-- Must remain a bare <script> (no defer) so window.* is set before defer-ed scripts run. --><script>window.RESEARCH_STATUS = Object.freeze({{ research_status_enum | tojson }});window.RESEARCH_TERMINAL_STATES = Object.freeze(new Set({{ research_terminal_states | tojson }}));+ window.LDR_LOG_LIMITS = Object.freeze({{ log_limits | tojson }});</script><!-- Constants and URL Configuration -->diff --git a/tests/utilities/test_log_utils.py b/tests/utilities/test_log_utils.pyindex d8869ef53e..528877f6af 100644--- a/tests/utilities/test_log_utils.py+++ b/tests/utilities/test_log_utils.py@@ -454,6 +454,92 @@ class TestDatabaseSink:mock_queue.put_nowait.assert_not_called()+class TestTruncateForDatabase:+ """Tests for _truncate_for_database helper and database_sink integration.++ Mirrors the cap discipline of _truncate_for_frontend: a long message is+ cut at DATABASE_MESSAGE_MAX_LENGTH chars and a short indicator is+ appended that reports the original length. Stops langgraph fetch logs+ (which inline 10 KB page bodies) from being persisted in full into+ ResearchLog rows.+ """++ def test_short_message_passes_through_unchanged(self):+ from local_deep_research.utilities.log_utils import (+ DATABASE_MESSAGE_MAX_LENGTH,+ _truncate_for_database,+ )++ short = "small payload"+ assert len(short) < DATABASE_MESSAGE_MAX_LENGTH+ assert _truncate_for_database(short) == short++ def test_message_at_exact_cap_is_not_truncated(self):+ """The cap is inclusive — a message of exactly the cap length must+ pass through unchanged."""+ from local_deep_research.utilities.log_utils import (+ DATABASE_MESSAGE_MAX_LENGTH,+ _truncate_for_database,+ )++ exact = "Y" * DATABASE_MESSAGE_MAX_LENGTH+ out = _truncate_for_database(exact)+ assert out == exact+ assert "truncated" not in out++ def test_long_message_truncated_with_indicator(self):+ from local_deep_research.utilities.log_utils import (+ DATABASE_MESSAGE_MAX_LENGTH,+ _truncate_for_database,+ )++ big = "X" * (DATABASE_MESSAGE_MAX_LENGTH + 5000)+ out = _truncate_for_database(big)+ assert out.startswith("X" * DATABASE_MESSAGE_MAX_LENGTH)+ assert "truncated" in out+ assert str(len(big)) in out # original length is surfaced+ # Indicator overhead is bounded (~100 chars), well under the cap.+ assert len(out) < DATABASE_MESSAGE_MAX_LENGTH + 200++ def test_database_sink_queues_truncated_message(self):+ """database_sink must apply the cap BEFORE queueing so the 10 KB+ blob never sits in _log_queue (bounded to 1000 entries; without the+ cap, that's a 10 MB worst-case transient)."""+ from local_deep_research.utilities.log_utils import (+ DATABASE_MESSAGE_MAX_LENGTH,+ database_sink,+ )++ big = "Z" * (DATABASE_MESSAGE_MAX_LENGTH + 1000)+ mock_message = Mock()+ mock_message.record = {+ "time": datetime.now(),+ "message": big,+ "name": "test_module",+ "function": "test_function",+ "line": 1,+ "level": Mock(name="INFO"),+ "extra": {"research_id": "rid-truncate"},+ }++ with patch(+ "local_deep_research.utilities.log_utils.has_app_context",+ return_value=False,+ ):+ with patch(+ "local_deep_research.utilities.log_utils._log_queue"+ ) as mock_queue:+ database_sink(mock_message)++ mock_queue.put_nowait.assert_called_once()+ queued = mock_queue.put_nowait.call_args[0][0]+ # The dict put on the queue carries the truncated string, not the+ # original 10KB+ blob.+ assert len(queued["message"]) < len(big)+ assert queued["message"].startswith("Z" * DATABASE_MESSAGE_MAX_LENGTH)+ assert "truncated" in queued["message"]++class TestFrontendProgressSink:"""Tests for frontend_progress_sink function."""diff --git a/tests/web/models/test_database.py b/tests/web/models/test_database.pyindex b7e9877f2e..423c83bcab 100644--- a/tests/web/models/test_database.py+++ b/tests/web/models/test_database.py@@ -235,6 +235,75 @@ class TestGetLogsForResearch:"newest",]+ def test_get_logs_with_limit_returns_newest_n_in_ascending_order_real_sqlite(+ self,+ ):+ """End-to-end SQL check against a real in-memory SQLite session.++ The mock-based ordering test above only proves Python's ``reversed()``+ is called — the mock returns a fixed list regardless of which+ ``order_by``/``limit`` was invoked. This test inserts 10 actual rows,+ runs the function through SQLAlchemy, and asserts that the SQL+ produces newest-first DESC + LIMIT and the caller sees the 3 newest+ rows in oldest-first order. Catches any regression where ``desc()``+ is dropped, ``limit()`` is moved, or ``reversed()`` is removed.+ """+ from datetime import datetime, timedelta, timezone++ from sqlalchemy import create_engine+ from sqlalchemy.orm import sessionmaker++ from local_deep_research.database.models import Base, ResearchLog+ from local_deep_research.web.models.database import (+ get_logs_for_research,+ )++ engine = create_engine("sqlite:///:memory:")+ Base.metadata.create_all(engine)+ Session = sessionmaker(bind=engine)+ session = Session()++ # 10 orphan log rows at 1-minute intervals. No parent ResearchHistory+ # row is created — research_id is nullable and SQLite does not enforce+ # FK constraints without PRAGMA foreign_keys=ON, so the insert is+ # valid and isolates this test from the parent-table schema.+ base_time = datetime(2025, 1, 1, 0, 0, 0, tzinfo=timezone.utc)+ for i in range(10):+ session.add(+ ResearchLog(+ research_id="test-rid",+ timestamp=base_time + timedelta(minutes=i),+ message=f"Log {i}",+ module="test",+ function="test",+ line_no=i,+ level="INFO",+ )+ )+ session.commit()++ try:+ with patch(+ "local_deep_research.web.models.database.get_user_db_session"+ ) as mock_get_session:+ mock_get_session.return_value.__enter__ = Mock(+ return_value=session+ )+ mock_get_session.return_value.__exit__ = Mock(+ return_value=False+ )++ result = get_logs_for_research("test-rid", limit=3)++ assert [r["message"] for r in result] == [+ "Log 7",+ "Log 8",+ "Log 9",+ ]+ finally:+ session.close()+ engine.dispose()+def test_get_logs_without_limit_does_not_call_limit(self):"""When limit is None, the function preserves legacy behaviour anddoes not invoke .limit() on the query (returns every row, asc)."""