Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(memory): cap persisted log message size + share /history/logs cap constants (#4240)

LearningCircuitc96449482f3f
diff --git a/changelog.d/+log-message-db-cap.bugfix.md b/changelog.d/+log-message-db-cap.bugfix.md
new file mode 100644
index 0000000000..3b89f8d15b
--- /dev/null
+++ b/changelog.d/+log-message-db-cap.bugfix.md
@@ -0,0 +1 @@
+Cap persisted log messages at 5000 chars so long langgraph runs no longer accumulate 10 KB ``ResearchLog`` rows. Mirrors PR #4004's frontend truncation discipline, now applied to the database sink. Also promotes the duplicated ``/history/logs`` pagination-cap literals into a single shared constant (``HISTORY_LOGS_HARD_CAP`` / ``HISTORY_LOGS_DEFAULT_LIMIT``, exposed to the log panel as ``window.LDR_LOG_LIMITS``) so the route clamp and the panel's DOM cap no longer drift. Full diagnostics remain available in stderr/file sinks.
diff --git a/src/local_deep_research/constants.py b/src/local_deep_research/constants.py
index 253760a31f..85ca490e6f 100644
--- a/src/local_deep_research/constants.py
+++ b/src/local_deep_research/constants.py
@@ -97,6 +97,14 @@ DEFAULT_LOCAL_SEARCH_TEXT_SEPARATORS_JSON = '["\\n\\n", "\\n", ". ", " ", ""]'
SNIPPET_LENGTH_SHORT = 250
SNIPPET_LENGTH_LONG = 500
+# --- /history/logs/<id> pagination caps ---
+# Default matches the frontend logpanel DOM cap (MAX_LOG_ENTRIES); the
+# hard cap is the ceiling the route clamps to so a client cannot force
+# an unbounded load. Shared with the frontend via inject_frontend_constants
+# (see web/app_factory.py) → window.LDR_LOG_LIMITS (see base.html).
+HISTORY_LOGS_DEFAULT_LIMIT = 500
+HISTORY_LOGS_HARD_CAP = 5000
+
# --- Research history collection ---
RESEARCH_HISTORY_COLLECTION_NAME = "History"
RESEARCH_HISTORY_COLLECTION_DESCRIPTION = (
diff --git a/src/local_deep_research/utilities/log_utils.py b/src/local_deep_research/utilities/log_utils.py
index 5d22e85b60..2e9393392d 100644
--- a/src/local_deep_research/utilities/log_utils.py
+++ b/src/local_deep_research/utilities/log_utils.py
@@ -47,6 +47,14 @@ Default log directory to use.
# bytes), so it can exceed this value by that fixed overhead.
FRONTEND_MESSAGE_MAX_LENGTH = 2000
+# Cap the size of messages persisted to ResearchLog. The DB column is
+# unbounded TEXT, so a long langgraph run can accumulate thousands of
+# 10 KB rows — paginated reads (PR #4037) hide the symptom but don't
+# stop the storage growth. Same prefix-plus-indicator semantics as
+# FRONTEND_MESSAGE_MAX_LENGTH; container-log/stderr/file sinks remain
+# unchanged so full diagnostics are still preserved out-of-band.
+DATABASE_MESSAGE_MAX_LENGTH = 5000
+
class InterceptHandler(logging.Handler):
"""
@@ -353,7 +361,7 @@ def database_sink(message: loguru.Message) -> None:
# Create log entry dict
log_entry = {
"timestamp": record["time"],
- "message": record["message"],
+ "message": _truncate_for_database(record["message"]),
"module": record["name"],
"function": record["function"],
"line_no": int(record["line"]),
@@ -377,6 +385,23 @@ def database_sink(message: loguru.Message) -> None:
_write_log_to_database(log_entry)
+def _truncate_for_database(message: str) -> str:
+ """Bound the persisted size of a log message.
+
+ ``DATABASE_MESSAGE_MAX_LENGTH`` is the preserved-prefix length;
+ the stored string is at most that plus a ~100-char suffix that
+ reports the original length so debug context is not lost. Full
+ messages remain available in container-log/stderr/file sinks.
+ """
+ if len(message) <= DATABASE_MESSAGE_MAX_LENGTH:
+ return message
+ suffix = (
+ f"… (truncated; full message in server logs; "
+ f"original length: {len(message)} chars)"
+ )
+ return message[:DATABASE_MESSAGE_MAX_LENGTH] + suffix
+
+
def _truncate_for_frontend(message: str) -> str:
"""Bound the wire size of an outbound log message.
diff --git a/src/local_deep_research/web/app_factory.py b/src/local_deep_research/web/app_factory.py
index bb4ca57624..8388b4930e 100644
--- a/src/local_deep_research/web/app_factory.py
+++ b/src/local_deep_research/web/app_factory.py
@@ -449,7 +449,11 @@ def apply_middleware(app):
# Source of truth: src/local_deep_research/constants.py::ResearchStatus
# Frontend helpers: src/local_deep_research/web/static/js/config/constants.js
# Template injection: src/local_deep_research/web/templates/base.html
- from ..constants import ResearchStatus
+ from ..constants import (
+ HISTORY_LOGS_DEFAULT_LIMIT,
+ HISTORY_LOGS_HARD_CAP,
+ ResearchStatus,
+ )
@app.context_processor
def inject_frontend_constants():
@@ -491,6 +495,10 @@ def apply_middleware(app):
return {
"research_status_enum": {m.name: m.value for m in ResearchStatus},
"research_terminal_states": [str(s) for s in terminal],
+ "log_limits": {
+ "default": HISTORY_LOGS_DEFAULT_LIMIT,
+ "hard_cap": HISTORY_LOGS_HARD_CAP,
+ },
"egress_scope": scope,
}
diff --git a/src/local_deep_research/web/routes/history_routes.py b/src/local_deep_research/web/routes/history_routes.py
index 8f1c47dfcb..94665e7d42 100644
--- a/src/local_deep_research/web/routes/history_routes.py
+++ b/src/local_deep_research/web/routes/history_routes.py
@@ -4,7 +4,11 @@ from flask import Blueprint, jsonify, request, session
from loguru import logger
from sqlalchemy import func
-from ...constants import ResearchStatus
+from ...constants import (
+ HISTORY_LOGS_DEFAULT_LIMIT,
+ HISTORY_LOGS_HARD_CAP,
+ ResearchStatus,
+)
from ...database.models import ResearchHistory
from ...database.models.library import Document as Document
from ...database.session_context import get_user_db_session
@@ -399,19 +403,19 @@ def get_markdown(research_id):
def get_research_logs(research_id):
"""Get logs for a specific research ID.
- Accepts ``?limit=N`` to bound the response size; default 500 matches
- the frontend's ``MAX_LOG_ENTRIES`` DOM cap. Clamped to ``[1, 5000]``
- so a client cannot force an unbounded load (a long langgraph run can
- persist thousands of 10 KB rows; pre-cap the route allocated ~150 MB
- transient on the server and Firefox parsed a ~50 MB JSON response).
+ Accepts ``?limit=N`` to bound the response size; the default matches
+ the frontend's ``MAX_LOG_ENTRIES`` DOM cap. Clamped to
+ ``[1, HISTORY_LOGS_HARD_CAP]`` so a client cannot force an unbounded
+ load (a long langgraph run can persist thousands of rows; pre-cap
+ the route allocated ~150 MB transient on the server and Firefox
+ parsed a ~50 MB JSON response).
"""
username = session["username"]
- # Per-request cap. 500 matches MAX_LOG_ENTRIES in logpanel.js; the
- # 5000 ceiling lets explicit log-download flows still get more rows
- # but stops accidental unbounded loads.
- limit = request.args.get("limit", default=500, type=int)
- limit = max(1, min(limit, 5000))
+ limit = request.args.get(
+ "limit", default=HISTORY_LOGS_DEFAULT_LIMIT, type=int
+ )
+ limit = max(1, min(limit, HISTORY_LOGS_HARD_CAP))
# First check if the research exists
with get_user_db_session(username) as db_session:
diff --git a/src/local_deep_research/web/static/js/components/logpanel.js b/src/local_deep_research/web/static/js/components/logpanel.js
index e63856b8c3..6b78d80088 100644
--- a/src/local_deep_research/web/static/js/components/logpanel.js
+++ b/src/local_deep_research/web/static/js/components/logpanel.js
@@ -17,8 +17,13 @@
normalizeTimestamps,
} = window.LdrLogHelpers;
- // Maximum number of log entries to keep in DOM to prevent unbounded growth
- const MAX_LOG_ENTRIES = 500;
+ // Maximum number of log entries to keep in DOM to prevent unbounded growth.
+ // Seeded from window.LDR_LOG_LIMITS (set in base.html from Python's
+ // HISTORY_LOGS_DEFAULT_LIMIT in constants.py) so this DOM cap and the
+ // shared pagination default come from one source instead of drifting.
+ // Falls back to 500 if the injection is missing (e.g. in a unit-test
+ // harness without templates).
+ const MAX_LOG_ENTRIES = window.LDR_LOG_LIMITS?.default ?? 500;
// Shared state for log panel
window._logPanelState ||= {
@@ -1088,9 +1093,13 @@
SafeLogger.warn('downloadLogs called without researchId; skipping');
return;
}
- // Download path requests the server-side hard cap so users still
- // get the full tail (server enforces a 5000-row clamp).
- fetchLogsForResearch(researchId, 5000).then((logData) => {
+ // Download path requests the shared hard cap (window.LDR_LOG_LIMITS,
+ // Python's HISTORY_LOGS_HARD_CAP) so users get the full tail. NOTE:
+ // the route this fetch hits (/api/research/<id>/logs) does not yet
+ // honor ?limit server-side, so this value is currently advisory; the
+ // /history/logs route clamps to the same ceiling when used.
+ const hardCap = window.LDR_LOG_LIMITS?.hard_cap ?? 5000;
+ fetchLogsForResearch(researchId, hardCap).then((logData) => {
// Create a blob with the logs data
const blob = new Blob([JSON.stringify(logData, null, 2)], { type: 'application/json' });
diff --git a/src/local_deep_research/web/templates/base.html b/src/local_deep_research/web/templates/base.html
index 7ccdb92ad8..7deccdebc9 100644
--- a/src/local_deep_research/web/templates/base.html
+++ b/src/local_deep_research/web/templates/base.html
@@ -218,9 +218,11 @@
<!-- Source: src/local_deep_research/constants.py::ResearchStatus -->
<!-- Injected by: src/local_deep_research/web/app_factory.py (inject_frontend_constants) -->
<!-- Consumed by: src/local_deep_research/web/static/js/config/constants.js (ResearchStates helpers) -->
+ <!-- Must remain a bare <script> (no defer) so window.* is set before defer-ed scripts run. -->
<script>
window.RESEARCH_STATUS = Object.freeze({{ research_status_enum | tojson }});
window.RESEARCH_TERMINAL_STATES = Object.freeze(new Set({{ research_terminal_states | tojson }}));
+ window.LDR_LOG_LIMITS = Object.freeze({{ log_limits | tojson }});
</script>
<!-- Constants and URL Configuration -->
diff --git a/tests/utilities/test_log_utils.py b/tests/utilities/test_log_utils.py
index d8869ef53e..528877f6af 100644
--- a/tests/utilities/test_log_utils.py
+++ b/tests/utilities/test_log_utils.py
@@ -454,6 +454,92 @@ class TestDatabaseSink:
mock_queue.put_nowait.assert_not_called()
+class TestTruncateForDatabase:
+ """Tests for _truncate_for_database helper and database_sink integration.
+
+ Mirrors the cap discipline of _truncate_for_frontend: a long message is
+ cut at DATABASE_MESSAGE_MAX_LENGTH chars and a short indicator is
+ appended that reports the original length. Stops langgraph fetch logs
+ (which inline 10 KB page bodies) from being persisted in full into
+ ResearchLog rows.
+ """
+
+ def test_short_message_passes_through_unchanged(self):
+ from local_deep_research.utilities.log_utils import (
+ DATABASE_MESSAGE_MAX_LENGTH,
+ _truncate_for_database,
+ )
+
+ short = "small payload"
+ assert len(short) < DATABASE_MESSAGE_MAX_LENGTH
+ assert _truncate_for_database(short) == short
+
+ def test_message_at_exact_cap_is_not_truncated(self):
+ """The cap is inclusive — a message of exactly the cap length must
+ pass through unchanged."""
+ from local_deep_research.utilities.log_utils import (
+ DATABASE_MESSAGE_MAX_LENGTH,
+ _truncate_for_database,
+ )
+
+ exact = "Y" * DATABASE_MESSAGE_MAX_LENGTH
+ out = _truncate_for_database(exact)
+ assert out == exact
+ assert "truncated" not in out
+
+ def test_long_message_truncated_with_indicator(self):
+ from local_deep_research.utilities.log_utils import (
+ DATABASE_MESSAGE_MAX_LENGTH,
+ _truncate_for_database,
+ )
+
+ big = "X" * (DATABASE_MESSAGE_MAX_LENGTH + 5000)
+ out = _truncate_for_database(big)
+ assert out.startswith("X" * DATABASE_MESSAGE_MAX_LENGTH)
+ assert "truncated" in out
+ assert str(len(big)) in out # original length is surfaced
+ # Indicator overhead is bounded (~100 chars), well under the cap.
+ assert len(out) < DATABASE_MESSAGE_MAX_LENGTH + 200
+
+ def test_database_sink_queues_truncated_message(self):
+ """database_sink must apply the cap BEFORE queueing so the 10 KB
+ blob never sits in _log_queue (bounded to 1000 entries; without the
+ cap, that's a 10 MB worst-case transient)."""
+ from local_deep_research.utilities.log_utils import (
+ DATABASE_MESSAGE_MAX_LENGTH,
+ database_sink,
+ )
+
+ big = "Z" * (DATABASE_MESSAGE_MAX_LENGTH + 1000)
+ mock_message = Mock()
+ mock_message.record = {
+ "time": datetime.now(),
+ "message": big,
+ "name": "test_module",
+ "function": "test_function",
+ "line": 1,
+ "level": Mock(name="INFO"),
+ "extra": {"research_id": "rid-truncate"},
+ }
+
+ with patch(
+ "local_deep_research.utilities.log_utils.has_app_context",
+ return_value=False,
+ ):
+ with patch(
+ "local_deep_research.utilities.log_utils._log_queue"
+ ) as mock_queue:
+ database_sink(mock_message)
+
+ mock_queue.put_nowait.assert_called_once()
+ queued = mock_queue.put_nowait.call_args[0][0]
+ # The dict put on the queue carries the truncated string, not the
+ # original 10KB+ blob.
+ assert len(queued["message"]) < len(big)
+ assert queued["message"].startswith("Z" * DATABASE_MESSAGE_MAX_LENGTH)
+ assert "truncated" in queued["message"]
+
+
class TestFrontendProgressSink:
"""Tests for frontend_progress_sink function."""
diff --git a/tests/web/models/test_database.py b/tests/web/models/test_database.py
index b7e9877f2e..423c83bcab 100644
--- a/tests/web/models/test_database.py
+++ b/tests/web/models/test_database.py
@@ -235,6 +235,75 @@ class TestGetLogsForResearch:
"newest",
]
+ def test_get_logs_with_limit_returns_newest_n_in_ascending_order_real_sqlite(
+ self,
+ ):
+ """End-to-end SQL check against a real in-memory SQLite session.
+
+ The mock-based ordering test above only proves Python's ``reversed()``
+ is called — the mock returns a fixed list regardless of which
+ ``order_by``/``limit`` was invoked. This test inserts 10 actual rows,
+ runs the function through SQLAlchemy, and asserts that the SQL
+ produces newest-first DESC + LIMIT and the caller sees the 3 newest
+ rows in oldest-first order. Catches any regression where ``desc()``
+ is dropped, ``limit()`` is moved, or ``reversed()`` is removed.
+ """
+ from datetime import datetime, timedelta, timezone
+
+ from sqlalchemy import create_engine
+ from sqlalchemy.orm import sessionmaker
+
+ from local_deep_research.database.models import Base, ResearchLog
+ from local_deep_research.web.models.database import (
+ get_logs_for_research,
+ )
+
+ engine = create_engine("sqlite:///:memory:")
+ Base.metadata.create_all(engine)
+ Session = sessionmaker(bind=engine)
+ session = Session()
+
+ # 10 orphan log rows at 1-minute intervals. No parent ResearchHistory
+ # row is created — research_id is nullable and SQLite does not enforce
+ # FK constraints without PRAGMA foreign_keys=ON, so the insert is
+ # valid and isolates this test from the parent-table schema.
+ base_time = datetime(2025, 1, 1, 0, 0, 0, tzinfo=timezone.utc)
+ for i in range(10):
+ session.add(
+ ResearchLog(
+ research_id="test-rid",
+ timestamp=base_time + timedelta(minutes=i),
+ message=f"Log {i}",
+ module="test",
+ function="test",
+ line_no=i,
+ level="INFO",
+ )
+ )
+ session.commit()
+
+ try:
+ with patch(
+ "local_deep_research.web.models.database.get_user_db_session"
+ ) as mock_get_session:
+ mock_get_session.return_value.__enter__ = Mock(
+ return_value=session
+ )
+ mock_get_session.return_value.__exit__ = Mock(
+ return_value=False
+ )
+
+ result = get_logs_for_research("test-rid", limit=3)
+
+ assert [r["message"] for r in result] == [
+ "Log 7",
+ "Log 8",
+ "Log 9",
+ ]
+ finally:
+ session.close()
+ engine.dispose()
+
def test_get_logs_without_limit_does_not_call_limit(self):
"""When limit is None, the function preserves legacy behaviour and
does not invoke .limit() on the query (returns every row, asc)."""