~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
ci(forensics): request arrival/duration logging for #4431 navigation-stall hunts (#4536)
LearningCircuitd98513da609a
diff --git a/changelog.d/4536.bugfix.md b/changelog.d/4536.bugfix.mdnew file mode 100644index 0000000000..c714764f04--- /dev/null+++ b/changelog.d/4536.bugfix.md@@ -0,0 +1 @@+Fixed intermittent ~60s UI freezes / "Navigation timeout" failures (#4431) caused by heavy third-party imports (matplotlib via the benchmarks package) and synchronous stderr logging blocking the dev server's request pipeline under load. matplotlib/optuna.visualization are now imported lazily (only when benchmark visualizations are generated), and the stderr log sink uses `enqueue=True` so logging never blocks on I/O while holding loguru's handler lock. Note: `enqueue=True` makes stderr logging asynchronous (a background writer thread), so on an abrupt crash the last few buffered log lines may be lost and ordering relative to other sinks can differ slightly.diff --git a/src/local_deep_research/benchmarks/comparison/evaluator.py b/src/local_deep_research/benchmarks/comparison/evaluator.pyindex c1d040186c..d1f315729d 100644--- a/src/local_deep_research/benchmarks/comparison/evaluator.py+++ b/src/local_deep_research/benchmarks/comparison/evaluator.py@@ -10,10 +10,8 @@ from datetime import datetime, UTCfrom pathlib import Pathfrom typing import Any, Dict, List, Optional-import matplotlib.pyplot as pltimport numpy as npfrom loguru import logger-from matplotlib.patches import Circle, RegularPolygonfrom local_deep_research.benchmarks.efficiency.resource_monitor import (ResourceMonitor,@@ -31,6 +29,35 @@ from local_deep_research.config.llm_config import get_llmfrom local_deep_research.config.search_config import get_searchfrom local_deep_research.search_system import AdvancedSearchSystem+# matplotlib is imported LAZILY via _ensure_plotting_loaded() inside the+# visualization helpers below — NOT at module level. A module-level+# `import matplotlib.pyplot` executes when this module is imported, and+# because benchmarks/__init__.py pulls this module in, that import ran on+# the server's import path. matplotlib's import is heavy and, under the+# 2-core CI runner's GIL/CPU starvation, stretched to ~60s while holding+# the import lock — freezing the whole werkzeug request pipeline (#4431).+# These comparison visualizations only run in explicit benchmark+# comparisons, never on a request path.+#+# Module-level placeholders so tests can @patch these names and so the+# loader can fill them in place. None until first real visualization.+plt = None+Circle = None+RegularPolygon = None+++def _ensure_plotting_loaded():+ """Import matplotlib into module globals on first use (see #4431).++ Early-returns if already loaded (or a test has patched plt) so it never+ clobbers mocks.+ """+ global plt, Circle, RegularPolygon+ if plt is not None:+ return+ import matplotlib.pyplot as plt+ from matplotlib.patches import Circle, RegularPolygon+def compare_configurations(query: str,@@ -411,6 +438,7 @@ def _create_comparison_visualizations(output_dir: Directory to save visualizationstimestamp: Timestamp string for filenames"""+ _ensure_plotting_loaded()# Check if there are successful resultssuccessful_results = [r@@ -514,6 +542,7 @@ def _create_metric_comparison_chart(title: Chart titleoutput_path: Path to save the chart"""+ _ensure_plotting_loaded()# Create figure with multiple subplots (one per metric)fig, axes = plt.subplots(len(metric_keys), 1, figsize=(12, 5 * len(metric_keys))@@ -580,6 +609,7 @@ def _create_spider_chart(config_names: Names of configurationsoutput_path: Path to save the chart"""+ _ensure_plotting_loaded()# Try to import the radar chart moduletry:from matplotlib.path import Path@@ -738,6 +768,7 @@ def _create_pareto_chart(results: List[Dict[str, Any]], output_path: str):results: List of configuration resultsoutput_path: Path to save the chart"""+ _ensure_plotting_loaded()# Extract quality and speed metricsquality_scores = []speed_scores = []diff --git a/src/local_deep_research/benchmarks/optimization/optuna_optimizer.py b/src/local_deep_research/benchmarks/optimization/optuna_optimizer.pyindex 883024f3d2..c70bf8efba 100644--- a/src/local_deep_research/benchmarks/optimization/optuna_optimizer.py+++ b/src/local_deep_research/benchmarks/optimization/optuna_optimizer.py@@ -16,12 +16,9 @@ from typing import Any, Callable, Dict, List, Optional, Tupleimport joblibimport numpy as npimport optuna-from optuna.visualization import (- plot_contour,- plot_optimization_history,- plot_param_importances,- plot_slice,-)++# (matplotlib / optuna.visualization are imported lazily — see+# _ensure_plotting_loaded below and #4431.)from local_deep_research.benchmarks.efficiency.speed_profiler import (SpeedProfiler,@@ -35,19 +32,58 @@ from local_deep_research.benchmarks.evaluators import (# Import benchmark evaluator components-# Try to import visualization libraries, but don't fail if not available-try:- import matplotlib.pyplot as plt- from matplotlib.lines import Line2D-- # We'll use matplotlib for plotting visualization results+# Visualization libraries (matplotlib + optuna.visualization) are imported+# LAZILY — see _ensure_plotting_loaded below and #4431. find_spec only+# probes availability; it does NOT execute/import the module, so importing+# this file (and therefore `local_deep_research.benchmarks`) never pays+# matplotlib's ~60s cold-import cost.+import importlib.util- PLOTTING_AVAILABLE = True-except ImportError:- PLOTTING_AVAILABLE = False+PLOTTING_AVAILABLE = importlib.util.find_spec("matplotlib") is not None+if not PLOTTING_AVAILABLE:logger.warning("Matplotlib not available, visualization will be limited")+# Module-level placeholders so tests can @patch these names and so the+# loader below can fill them in place. None until first real visualization.+plt = None+Line2D = None+plot_contour = None+plot_optimization_history = None+plot_param_importances = None+plot_slice = None+++def _ensure_plotting_loaded():+ """Import matplotlib + optuna.visualization into module globals on first+ use.++ Deferred from module load on purpose: matplotlib's import is heavy and,+ under the 2-core CI runner's GIL/CPU starvation, stretched to ~60s while+ holding the import lock. Because benchmarks/__init__.py imports this+ module, a module-level matplotlib import froze the whole werkzeug request+ pipeline the first time any request touched `local_deep_research.benchmarks`+ — the flaky UI-shard navigation timeouts (#4431). The+ _create_*_visualizations methods call this after their PLOTTING_AVAILABLE+ guard; visualization only happens in benchmark-optimization runs, never+ on a request path. Early-returns if already loaded (or a test has patched+ plt) so it never clobbers mocks.+ """+ global plt, Line2D+ global plot_contour, plot_optimization_history+ global plot_param_importances, plot_slice+ if plt is not None:+ return+ import matplotlib.pyplot as plt+ from matplotlib.lines import Line2D+ from optuna.visualization import (+ plot_contour,+ plot_optimization_history,+ plot_param_importances,+ plot_slice,+ )++class OptunaOptimizer:"""Optimize parameters for Local Deep Research using Optuna.@@ -594,6 +630,7 @@ class OptunaOptimizer:"Matplotlib not available, skipping visualization creation")return+ _ensure_plotting_loaded()if not self.study or len(self.study.trials) < 2:logger.warning("Not enough trials to create visualizations")@@ -620,6 +657,7 @@ class OptunaOptimizer:or len(self.study.trials) < 2):return+ _ensure_plotting_loaded()# Create directory for visualizations_quick_viz_dir_path = Path(self.output_dir) / "visualizations"@@ -645,8 +683,9 @@ class OptunaOptimizer:Args:viz_dir: Directory to save visualizations"""- if not self.study:+ if not self.study or not PLOTTING_AVAILABLE:return+ _ensure_plotting_loaded()study = self.studytimestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")diff --git a/src/local_deep_research/utilities/log_utils.py b/src/local_deep_research/utilities/log_utils.pyindex 061b235be3..5d22e85b60 100644--- a/src/local_deep_research/utilities/log_utils.py+++ b/src/local_deep_research/utilities/log_utils.py@@ -563,7 +563,19 @@ def config_logger(name: str, debug: bool = False) -> None:# credential-bearing exception handler app-wide against the# frame-locals leak, independent of per-site logging discipline# (#4182).- logger.add(sys.stderr, level=stderr_level, diagnose=diagnose)+ #+ # enqueue=True on stderr: loguru emits to an in-memory queue and a+ # single background thread does the actual stderr write, so a log call+ # never blocks on stderr I/O while holding the handler's lock. Without+ # it, under the werkzeug threading dev server every request thread logs+ # synchronously, and when stderr back-pressures (e.g. a slow/full+ # `docker logs` pipe in CI) the lock-holder blocks mid-write and ALL+ # logging threads — i.e. all request threads — pile up behind the lock,+ # freezing the whole request pipeline for ~60s (#4431). Captured+ # forensically: 3/5 server threads parked in loguru's _protected_lock+ # under load. The database/progress sinks keep their own+ # emitting-thread context capture and are left synchronous.+ logger.add(sys.stderr, level=stderr_level, diagnose=diagnose, enqueue=True)logger.add(database_sink, level="DEBUG", diagnose=False)logger.add(frontend_progress_sink, diagnose=False)diff --git a/src/local_deep_research/web/app_factory.py b/src/local_deep_research/web/app_factory.pyindex 1fa02a970f..bb4ca57624 100644--- a/src/local_deep_research/web/app_factory.py+++ b/src/local_deep_research/web/app_factory.py@@ -131,6 +131,15 @@ def create_app():)app.wsgi_app = ServerHeaderMiddleware(app.wsgi_app) # type: ignore[method-assign]+ # CI/test-only request forensics for the #4431 navigation-stall hunts:+ # logs every request's arrival + duration so a silent log window can+ # be attributed to "request never arrived" vs "request hung in app".+ if os.environ.get("CI") or os.environ.get("TESTING"):+ from .utils.request_timing import RequestTimingMiddleware++ app.wsgi_app = RequestTimingMiddleware(app.wsgi_app) # type: ignore[method-assign]+ logger.info("Request-timing forensics middleware enabled (CI/TESTING)")+# App configuration# Generate or load a unique SECRET_KEY per installationimport secretsdiff --git a/src/local_deep_research/web/static/js/services/socket.js b/src/local_deep_research/web/static/js/services/socket.jsindex 7a0842a663..7169e5bcb2 100644--- a/src/local_deep_research/web/static/js/services/socket.js+++ b/src/local_deep_research/web/static/js/services/socket.js@@ -802,15 +802,42 @@ window.socket = (function() {return usingPolling;}+ // Auto-connect on page load for realtime pages — EXCEPT the chat page,+ // which connects lazily (on the first subscribeToResearch) instead.+ //+ // Why chat is special (#4431): the chat tests, and real users, navigate+ // to/from /chat/ frequently. Eagerly opening a Socket.IO connection on+ // every /chat/ load creates a churn of engine.io connect/disconnect+ // cycles. On the werkzeug dev server (Flask-SocketIO threading mode, no+ // eventlet/gevent) that churn, under CPU pressure, freezes the whole+ // WSGI request pipeline for ~60s — the flaky UI-shard navigation+ // timeouts. Chat doesn't need the socket until a research actually+ // streams: chat.js calls subscribeToResearch (which lazily initializes+ // the socket) on send and when resuming an in-progress research, and+ // has an HTTP polling backup either way. Other realtime pages+ // (/research, /progress, /benchmark) keep eager connect — they aren't+ // navigated in the same churny way.+ function autoInitSocket() {+ // Match the chat page (/chat/ and /chat/<session_id>) precisely —+ // not a loose substring that would also catch hypothetical paths+ // like /chat-archive/.+ const path = window.location.pathname;+ if (path === '/chat' || path.startsWith('/chat/')) {+ SafeLogger.log('Socket.IO: deferring connect on chat page (lazy on subscribe)');+ return;+ }+ initializeSocket();+ }+// Initialize socket only after DOM is ready to avoid blocking DOMContentLoaded detection// This is important for Puppeteer tests that use waitUntil: 'domcontentloaded'if (document.readyState === 'loading') {document.addEventListener('DOMContentLoaded', function() {- setTimeout(initializeSocket, 100);+ setTimeout(autoInitSocket, 100);});} else {// DOM already ready- setTimeout(initializeSocket, 100);+ setTimeout(autoInitSocket, 100);}// Expose functions globallydiff --git a/src/local_deep_research/web/utils/request_timing.py b/src/local_deep_research/web/utils/request_timing.pynew file mode 100644index 0000000000..f464b0f0c7--- /dev/null+++ b/src/local_deep_research/web/utils/request_timing.py@@ -0,0 +1,113 @@+"""Request-arrival/duration forensics for CI test runs (issue #4431).++The UI test shards intermittently fail with 60-second navigation+timeouts, and the server logs go silent for the same window — but the+app only logs explicit events, so a silent window cannot distinguish+"the request never reached the server" (connection-level stall: listen+backlog, docker-proxy, browser socket pool starved by engine.io polls)+from "the request reached Flask and hung" (app-level stall: lock, DB+pool, GIL hog).++This middleware settles that by logging every request's arrival and its+WSGI-call duration. It is wired up by app_factory ONLY when CI or+TESTING is set, so production logging is unaffected.++Log format (kept compact — engine.io polls arrive every ~5s/client):+ [req] > GET /chat/+ [req] < GET /chat/ 0.04s+Slow completions get a WARNING with the duration, which the CI workflow+log-grep surfaces.++Freeze thread-dump (dead-man's switch)+--------------------------------------+The arrival log proves *that* the pipeline froze, but not *what* it was+stuck on. So this middleware also arms ``faulthandler.dump_traceback_later``+and re-arms it on every request arrival. If no request arrives for+``FREEZE_DUMP_SECONDS`` (i.e. a freeze), faulthandler dumps ALL thread+stacks to stderr — and because it runs on a dedicated C timer thread it+fires even when the GIL is starved, which a Python watchdog thread could+not. During a ~60s freeze this yields 2-3 dumps showing exactly which+threads are blocked (werkzeug accept loop? a lock? a DB/SQLCipher call?+the scheduler?). Healthy operation re-arms the timer faster than it+fires, so no dumps appear. Captured in the CI server-log artifact.+"""++import faulthandler+import sys+import time++from loguru import logger++# Above this, completion is logged as a warning — the interesting cases.+SLOW_REQUEST_SECONDS = 2.0++# No request for this long ⇒ assume a freeze and dump all thread stacks.+# Smaller than the 60s navigation timeout so a freeze produces 2-3 dumps,+# larger than legitimate inter-test idle so healthy runs stay quiet-ish.+FREEZE_DUMP_SECONDS = 20.0+++def _should_arm_freeze_dump():+ """Arm the dead-man's switch only for the real, long-running server.++ create_app() runs thousands of times under pytest (with CI=true), and+ arming a repeating faulthandler dump in each would spew stack traces+ across the whole pytest run. The freeze we care about only happens on+ the live UI-shard server, so skip arming when pytest is in the process.+ """+ return "pytest" not in sys.modules+++def _arm_freeze_dump():+ if not _should_arm_freeze_dump():+ return+ try:+ faulthandler.enable()+ faulthandler.dump_traceback_later(+ FREEZE_DUMP_SECONDS, repeat=True, file=sys.stderr+ )+ except Exception as exc: # noqa: silent-exception+ # Diagnostics must never take the server down.+ logger.debug(f"freeze thread-dump arm failed: {exc}")+++class RequestTimingMiddleware:+ """Outermost WSGI wrapper that logs request arrival and duration.++ Duration covers the WSGI call (view execution), not response+ streaming — for stall forensics the arrival line is the signal that+ matters: its absence during a navigation timeout proves the request+ never reached the WSGI layer.+ """++ def __init__(self, wsgi_app):+ self.wsgi_app = wsgi_app+ # Arm the freeze thread-dump dead-man's switch (no-op under pytest).+ _arm_freeze_dump()++ def __call__(self, environ, start_response):+ # Re-arm the dead-man's switch: as long as requests keep arriving+ # the dump never fires; a freeze (no arrivals) lets it fire and+ # capture the stuck thread stacks.+ _arm_freeze_dump()++ method = environ.get("REQUEST_METHOD", "-")+ path = environ.get("PATH_INFO", "-")+ # engine.io transport/sid make poll churn correlatable. (sid is+ # logged on purpose for correlation; logs are CI-only artifacts.)+ if path.startswith("/socket.io"):+ query = environ.get("QUERY_STRING", "")+ path = f"{path}?{query}" if query else path+ # Strip CR/LF so a crafted PATH_INFO/QUERY_STRING can't inject fake+ # log lines (the forensics output is grep'd downstream).+ path = path.replace("\r", "\\r").replace("\n", "\\n")+ logger.info(f"[req] > {method} {path}")+ start = time.monotonic()+ try:+ return self.wsgi_app(environ, start_response)+ finally:+ elapsed = time.monotonic() - start+ if elapsed >= SLOW_REQUEST_SECONDS:+ logger.warning(f"[req] < {method} {path} {elapsed:.1f}s SLOW")+ else:+ logger.info(f"[req] < {method} {path} {elapsed:.2f}s")diff --git a/tests/js/services/socket_lazy_chat.test.js b/tests/js/services/socket_lazy_chat.test.jsnew file mode 100644index 0000000000..fedb6c1e76--- /dev/null+++ b/tests/js/services/socket_lazy_chat.test.js@@ -0,0 +1,72 @@+/**+ * Tests for the #4431 fix: Socket.IO connects lazily on /chat/.+ *+ * The chat page must NOT open a Socket.IO connection on page load — that+ * per-navigation connect/disconnect churn freezes the werkzeug dev server.+ * Other realtime pages keep their eager auto-connect.+ *+ * We drive the module's auto-init (a setTimeout scheduled at import time)+ * with fake timers and assert whether io() — the connection factory — was+ * called, per page path. resetModules + dynamic import gives each case a+ * fresh evaluation of the IIFE with its own window.location.pathname.+ */+import { beforeEach, afterEach, describe, it, expect, vi } from 'vitest';++function setupEnv(pathname) {+ vi.resetModules();+ Object.defineProperty(window, 'location', {+ configurable: true,+ value: { pathname, protocol: 'http:', host: 'localhost' },+ });+ const io = vi.fn(() => ({+ on: vi.fn(),+ emit: vi.fn(),+ off: vi.fn(),+ connected: false,+ }));+ globalThis.io = io;+ window.api = { getResearchStatus: vi.fn(), getCsrfToken: () => '' };+ window.ResearchStates = { isTerminal: () => false, logLevel: () => 'info' };+ return io;+}++describe('Socket.IO auto-connect gating (#4431)', () => {+ beforeEach(() => vi.useFakeTimers());+ afterEach(() => {+ vi.useRealTimers();+ delete globalThis.io;+ });++ it('does NOT auto-connect on a /chat/ page (lazy)', async () => {+ const io = setupEnv('/chat/some-session-id');+ await import('@js/services/socket.js');+ // Fire the auto-init setTimeout(…, 100) scheduled at import.+ vi.advanceTimersByTime(300);+ expect(io).not.toHaveBeenCalled();+ });++ it('DOES auto-connect on a /progress/ page (eager, unchanged)', async () => {+ const io = setupEnv('/progress/abc-123');+ await import('@js/services/socket.js');+ vi.advanceTimersByTime(300);+ expect(io).toHaveBeenCalledTimes(1);+ });++ it('DOES auto-connect on a /research page (eager, unchanged)', async () => {+ const io = setupEnv('/research');+ await import('@js/services/socket.js');+ vi.advanceTimersByTime(300);+ expect(io).toHaveBeenCalledTimes(1);+ });++ it('still connects lazily on /chat/ when a research is subscribed', async () => {+ const io = setupEnv('/chat/some-session-id');+ await import('@js/services/socket.js');+ vi.advanceTimersByTime(300);+ expect(io).not.toHaveBeenCalled(); // confirmed lazy at load++ // Subscribing to a research must initialize the socket on demand.+ window.socket.subscribeToResearch('research-1', () => {});+ expect(io).toHaveBeenCalledTimes(1);+ });+});diff --git a/tests/web/utils/test_request_timing.py b/tests/web/utils/test_request_timing.pynew file mode 100644index 0000000000..d517765ffe--- /dev/null+++ b/tests/web/utils/test_request_timing.py@@ -0,0 +1,186 @@+"""Tests for the CI request-timing forensics middleware (#4431)."""++from unittest.mock import MagicMock++import pytest++from local_deep_research.web.utils.request_timing import (+ FREEZE_DUMP_SECONDS,+ RequestTimingMiddleware,+ SLOW_REQUEST_SECONDS,+ _should_arm_freeze_dump as _orig_should_arm_freeze_dump,+)++[email protected](autouse=True)+def _mock_faulthandler(monkeypatch):+ """Replace faulthandler so tests never arm a real repeating thread-dump+ timer (which would fire on stderr mid-suite), and force the pytest gate+ OPEN so the arming path is actually exercised. Returns the mock so a+ test can assert the dead-man's switch was armed."""+ from local_deep_research.web.utils import request_timing as mod++ fake = MagicMock()+ monkeypatch.setattr(mod, "faulthandler", fake)+ # _should_arm_freeze_dump() returns False under pytest; override so the+ # arming path runs in these tests.+ monkeypatch.setattr(mod, "_should_arm_freeze_dump", lambda: True)+ return fake+++def _make_environ(method="GET", path="/chat/", query=""):+ return {+ "REQUEST_METHOD": method,+ "PATH_INFO": path,+ "QUERY_STRING": query,+ }+++def _collect_logs(monkeypatch):+ """Capture loguru records emitted by the middleware module."""+ records = []+ from local_deep_research.web.utils import request_timing as mod++ class FakeLogger:+ def info(self, msg):+ records.append(("info", msg))++ def warning(self, msg):+ records.append(("warning", msg))++ def debug(self, msg):+ records.append(("debug", msg))++ monkeypatch.setattr(mod, "logger", FakeLogger())+ return records+++def test_logs_arrival_and_completion(monkeypatch):+ records = _collect_logs(monkeypatch)++ def app(environ, start_response):+ return [b"ok"]++ mw = RequestTimingMiddleware(app)+ body = mw(_make_environ(), lambda *a: None)++ assert body == [b"ok"]+ assert records[0] == ("info", "[req] > GET /chat/")+ level, msg = records[1]+ assert level == "info"+ assert msg.startswith("[req] < GET /chat/")+++def test_slow_request_logged_as_warning(monkeypatch):+ records = _collect_logs(monkeypatch)+ from local_deep_research.web.utils import request_timing as mod++ # Simulate a slow WSGI call without sleeping: jump the clock by+ # patching time.monotonic to advance past the slow threshold.+ ticks = iter([100.0, 100.0 + SLOW_REQUEST_SECONDS + 1])+ monkeypatch.setattr(mod.time, "monotonic", lambda: next(ticks))++ mw = RequestTimingMiddleware(lambda e, s: [b"ok"])+ mw(_make_environ(path="/slow"), lambda *a: None)++ level, msg = records[-1]+ assert level == "warning"+ assert "SLOW" in msg+++def test_socketio_paths_keep_query_string(monkeypatch):+ records = _collect_logs(monkeypatch)++ mw = RequestTimingMiddleware(lambda e, s: [b"ok"])+ mw(+ _make_environ(path="/socket.io/", query="transport=polling&sid=abc"),+ lambda *a: None,+ )++ assert records[0] == (+ "info",+ "[req] > GET /socket.io/?transport=polling&sid=abc",+ )+++def test_arrival_logged_even_when_app_raises(monkeypatch):+ records = _collect_logs(monkeypatch)++ def exploding_app(environ, start_response):+ raise RuntimeError("boom")++ mw = RequestTimingMiddleware(exploding_app)+ try:+ mw(_make_environ(path="/boom"), lambda *a: None)+ except RuntimeError:+ pass++ assert records[0] == ("info", "[req] > GET /boom")+ # completion line still emitted by the finally block+ assert any("[req] < GET /boom" in msg for _, msg in records[1:])+++def test_arms_freeze_dump_on_init_and_rearms_per_request(+ monkeypatch, _mock_faulthandler+):+ _collect_logs(monkeypatch)++ # Construction arms the dead-man's switch.+ mw = RequestTimingMiddleware(lambda e, s: [b"ok"])+ _mock_faulthandler.enable.assert_called_once()+ assert _mock_faulthandler.dump_traceback_later.call_count == 1+ _, kwargs = _mock_faulthandler.dump_traceback_later.call_args+ assert _mock_faulthandler.dump_traceback_later.call_args.args[0] == (+ FREEZE_DUMP_SECONDS+ )+ assert kwargs.get("repeat") is True++ # Each request re-arms it (resets the countdown).+ mw(_make_environ(), lambda *a: None)+ assert _mock_faulthandler.dump_traceback_later.call_count == 2+++def test_path_crlf_is_sanitized_against_log_injection(monkeypatch):+ records = _collect_logs(monkeypatch)++ mw = RequestTimingMiddleware(lambda e, s: [b"ok"])+ mw(_make_environ(path="/evil\n[req] > GET /forged"), lambda *a: None)++ arrival = records[0][1]+ assert "\n" not in arrival+ assert "\\n" in arrival+ assert arrival == "[req] > GET /evil\\n[req] > GET /forged"+++def test_freeze_dump_failure_never_breaks_requests(monkeypatch):+ """If faulthandler raises, the request must still be served."""+ from local_deep_research.web.utils import request_timing as mod++ boom = MagicMock()+ boom.enable.side_effect = RuntimeError("no faulthandler")+ boom.dump_traceback_later.side_effect = RuntimeError("nope")+ monkeypatch.setattr(mod, "faulthandler", boom)+ monkeypatch.setattr(mod, "_should_arm_freeze_dump", lambda: True)+ _collect_logs(monkeypatch)++ mw = RequestTimingMiddleware(lambda e, s: [b"served"])+ assert mw(_make_environ(), lambda *a: None) == [b"served"]+++def test_freeze_dump_not_armed_under_pytest(monkeypatch):+ """The dead-man's switch must be a no-op under pytest so it doesn't spew+ dumps across the test run; only the real server arms it."""+ from local_deep_research.web.utils import request_timing as mod++ fake = MagicMock()+ monkeypatch.setattr(mod, "faulthandler", fake)+ # Undo the autouse fixture's gate override; exercise the REAL gate,+ # which sees pytest in sys.modules and must suppress arming.+ monkeypatch.setattr(+ mod, "_should_arm_freeze_dump", _orig_should_arm_freeze_dump+ )+ assert mod._should_arm_freeze_dump() is False+ mw = RequestTimingMiddleware(lambda e, s: [b"ok"])+ mw(_make_environ(), lambda *a: None)+ fake.dump_traceback_later.assert_not_called()+ fake.enable.assert_not_called()