~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Census fence: the FastAPI Jinja environment must expose everything the
templates actually reference, and inject the same default context Flask's
``render_template_with_defaults`` + ``@app.context_processor`` did.
Why a census and not a spot-check
---------------------------------
Under Flask the templates rendered against ``app.jinja_env`` plus a
context processor. The port replaced that with
``web/template_config.py`` (``_LDRTemplates``) and
``web/dependencies/template_helpers.render_template``, with the globals
registered in ``fastapi_app._setup_template_globals()``. A helper that
did not make the trip is invisible until a user opens the page: Jinja's
default ``Undefined`` renders an empty string for ``{{ x }}`` but raises
``UndefinedError`` (a 500) the moment the value is *called*, iterated,
or filtered. Two helpers on this path (``effective_scope_for_display``
and ``normalize_provider``) were already found missing and restored
during this migration, so the defect class is live.
So instead of naming the helpers we expect, these tests *derive* the
required set from the template sources:
* :func:`_called_global_names` walks every template's Jinja AST and
collects every name that is **called** and not bound locally (macro,
``{% import %}``, ``{% set %}``, loop target). Anything in that set
must be in ``templates.env.globals`` or the page 500s.
* the ``StrictUndefined`` render fences take the whole render graph of
the always-rendered pages (``base.html`` and the three auth pages)
and turn *every* unresolved name -- callable or not -- into a hard
error, so a dropped context default cannot hide behind lenient
``Undefined``.
Everything else here pins observable behaviour of a real render:
autoescaping, the CSRF token, flash across a redirect, and each value
Flask injected into every template.
"""
import contextlib
import json
import re
from pathlib import Path
import jinja2
import pytest
from jinja2 import nodes
from starlette.requests import Request
TEMPLATES_DIR = (
Path(__file__).resolve().parents[2]
/ "src/local_deep_research/web/templates"
)
# Provided by Jinja itself inside a `{% call %}` block, not by the
# environment; the census must not demand it from env.globals.
_JINJA_BUILTIN_CALLABLES = {"caller"}
# ---------------------------------------------------------------------------
# Census extraction
# ---------------------------------------------------------------------------
def _locally_bound_names(ast: nodes.Template) -> set[str]:
"""Names a template binds itself: macros, imports, {% set %}, loops."""
bound: set[str] = set()
for node in ast.find_all(nodes.FromImport):
for name in node.names:
bound.add(name[1] if isinstance(name, tuple) else name)
for node in ast.find_all(nodes.Import):
bound.add(node.target)
for node in ast.find_all(nodes.Macro):
bound.add(node.name)
for node in ast.find_all(nodes.Assign):
if isinstance(node.target, nodes.Name):
bound.add(node.target.name)
for node in ast.find_all(nodes.With):
for target in node.targets:
if isinstance(target, nodes.Name):
bound.add(target.name)
for node in ast.find_all(nodes.For):
target = node.target
if isinstance(target, nodes.Name):
bound.add(target.name)
elif isinstance(target, nodes.Tuple):
for item in target.items:
if isinstance(item, nodes.Name):
bound.add(item.name)
return bound
def _called_global_names() -> dict[str, set[str]]:
"""Map every called-but-unbound name to the templates that call it.
A name in this map is invoked as ``{{ name(...) }}`` somewhere. If
the environment does not supply it, Jinja hands back ``Undefined``
and calling it raises ``UndefinedError`` -- a 500 on a page that
renders fine on main.
"""
env = jinja2.Environment( # noqa: S701 - parse only, never rendered
loader=jinja2.FileSystemLoader(str(TEMPLATES_DIR)),
)
found: dict[str, set[str]] = {}
for path in sorted(TEMPLATES_DIR.rglob("*.html")):
rel = path.relative_to(TEMPLATES_DIR).as_posix()
ast = env.parse(path.read_text(encoding="utf-8"), filename=rel)
bound = _locally_bound_names(ast)
for call in ast.find_all(nodes.Call):
fn = call.node
if not isinstance(fn, nodes.Name):
continue # obj.method() -- resolved on the object
if fn.name in bound or fn.name in _JINJA_BUILTIN_CALLABLES:
continue
found.setdefault(fn.name, set()).add(rel)
return found
def _referenced_filters() -> dict[str, set[str]]:
env = jinja2.Environment( # noqa: S701 - parse only, never rendered
loader=jinja2.FileSystemLoader(str(TEMPLATES_DIR)),
)
found: dict[str, set[str]] = {}
for path in sorted(TEMPLATES_DIR.rglob("*.html")):
rel = path.relative_to(TEMPLATES_DIR).as_posix()
ast = env.parse(path.read_text(encoding="utf-8"), filename=rel)
for node in ast.find_all(nodes.Filter):
found.setdefault(node.name, set()).add(rel)
for node in ast.find_all(nodes.Test):
found.setdefault(node.name, set()).add(rel)
return found
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture(scope="module")
def app_and_templates():
"""The real app (its import runs _setup_template_globals) + templates."""
from local_deep_research.web import fastapi_app
from local_deep_research.web.template_config import templates
return fastapi_app.app, templates
@pytest.fixture
def make_request(app_and_templates):
"""Build a Starlette Request with a session, as SessionMiddleware would."""
app, _ = app_and_templates
def _make(session: dict | None = None) -> Request:
scope = {
"type": "http",
"http_version": "1.1",
"method": "GET",
"path": "/",
"raw_path": b"/",
"root_path": "",
"query_string": b"",
"headers": [(b"host", b"testserver")],
"scheme": "http",
"server": ("testserver", 80),
"client": ("testclient", 1234),
"app": app,
"router": app.router,
"session": {} if session is None else session,
}
return Request(scope)
return _make
@contextlib.contextmanager
def _strict_undefined(templates):
"""Temporarily make every unresolved name a hard error.
Jinja reads ``environment.undefined`` at render time, but compiled
templates are cached, so the cache is cleared on both edges.
"""
env = templates.env
previous = env.undefined
env.undefined = jinja2.StrictUndefined
env.cache.clear()
try:
yield
finally:
env.undefined = previous
env.cache.clear()
def _render(templates, request, name, context=None):
response = templates.TemplateResponse(
request=request, name=name, context=dict(context or {})
)
return response.body.decode("utf-8")
# ---------------------------------------------------------------------------
# 1. The census itself
# ---------------------------------------------------------------------------
def test_census_extractor_finds_the_known_helper_calls():
"""Guard the guard: if the AST walk silently stopped finding names,
every census assertion below would pass vacuously. These seven are
visibly called in the checked-in templates."""
called = _called_global_names()
assert {
"csrf_token",
"get_flashed_messages",
"get_theme_metadata",
"get_themes_json",
"url_for",
"vite_asset",
"vite_hmr",
} <= set(called), sorted(called)
def test_census_extractor_excludes_locally_defined_macros():
"""`help_panel` & friends are imported macros, not env globals --
demanding them from env.globals would make the census unusable."""
called = _called_global_names()
for macro in ("help_panel", "help_tip", "help_step", "render_dropdown"):
assert macro not in called, f"{macro} wrongly treated as a global"
def test_every_called_template_name_is_registered_as_a_jinja_global(
app_and_templates,
):
"""The census: everything the templates invoke must exist in the
environment. A name missing here is an UndefinedError -- a 500 -- on
every page that calls it."""
_, templates = app_and_templates
globals_ = templates.env.globals
called = _called_global_names()
missing = {
name: sorted(users)
for name, users in called.items()
if name not in globals_
}
assert not missing, (
"template calls names the Jinja environment does not provide "
f"(each is a 500 on those pages): {missing}"
)
def test_every_referenced_filter_and_test_is_registered(app_and_templates):
"""Same census for `|filters` and `is tests`. Flask ships a handful
beyond Jinja's defaults (e.g. `tojson`); a missing one is a
TemplateAssertionError at render."""
_, templates = app_and_templates
env = templates.env
known = set(env.filters) | set(env.tests)
referenced = _referenced_filters()
assert "tojson" in referenced, "expected base.html to still use |tojson"
missing = {
name: sorted(users)
for name, users in referenced.items()
if name not in known
}
assert not missing, f"templates use unregistered filters/tests: {missing}"
# ---------------------------------------------------------------------------
# 2. Whole-render-graph fences (StrictUndefined)
# ---------------------------------------------------------------------------
# Pages rendered on essentially every request, with only the context a
# route genuinely owns. Everything else must come from the environment
# or from the automatic default-context injection.
_STRICT_RENDER_CASES = [
("base.html", {"active_page": "research"}),
(
"auth/login.html",
{"allow_registrations": True, "next_page": None},
),
("auth/register.html", {"password_requirements": {}}),
("auth/change_password.html", {"password_requirements": {}}),
("settings_dashboard.html", {"active_page": "settings"}),
]
@pytest.mark.parametrize(("name", "context"), _STRICT_RENDER_CASES)
def test_render_graph_resolves_every_name(
app_and_templates, make_request, name, context
):
"""Render the full graph with StrictUndefined so *any* name the
environment or default context fails to supply raises instead of
silently emitting an empty string."""
_, templates = app_and_templates
request = make_request({"username": "census_user"})
with _strict_undefined(templates):
html = _render(templates, request, name, context)
assert "<html" in html.lower()
# ---------------------------------------------------------------------------
# 3. Autoescaping
# ---------------------------------------------------------------------------
def test_autoescape_is_on_for_every_template_file(app_and_templates):
"""Flask autoescapes every .html template. Census the tree so a
template added with an extension outside the autoescape set (.jinja,
.txt) cannot ship unescaped."""
_, templates = app_and_templates
autoescape = templates.env.autoescape
unescaped = [
p.relative_to(TEMPLATES_DIR).as_posix()
for p in sorted(TEMPLATES_DIR.rglob("*"))
if p.is_file()
and not autoescape(p.relative_to(TEMPLATES_DIR).as_posix())
]
assert not unescaped, f"templates rendered unescaped: {unescaped}"
def test_context_values_are_html_escaped_in_a_real_render(
app_and_templates, make_request
):
"""base.html stamps the session username into two meta tags. With
autoescaping off this is stored XSS on every page."""
_, templates = app_and_templates
payload = '<script>alert("xss")</script>'
request = make_request({"username": payload})
html = _render(templates, request, "base.html", {})
assert payload not in html
meta = re.search(r'<meta name="user-id" content="([^"]*)"', html)
assert meta is not None, "user-id meta tag disappeared from base.html"
assert "<script>" in meta.group(1)
# ---------------------------------------------------------------------------
# 4. CSRF token injection
# ---------------------------------------------------------------------------
def _csrf_from(html: str) -> str:
match = re.search(r'<meta name="csrf-token" content="([^"]*)"', html)
assert match is not None, "base.html lost its csrf-token meta tag"
return match.group(1)
def test_csrf_token_is_injected_and_accepted_by_the_validator(
app_and_templates, make_request
):
"""The rendered token must be the session's token -- not the empty
string the module-level `csrf_token()` global returns -- and must
pass the real validator the CSRF middleware uses."""
from local_deep_research.web.dependencies.csrf import validate_csrf_token
_, templates = app_and_templates
session: dict = {}
request = make_request(session)
token = _csrf_from(_render(templates, request, "base.html", {}))
assert token, (
"csrf_token() rendered empty -- the per-render override never ran, "
"so every form on the page carries a token the middleware rejects"
)
assert token == session.get("_csrf_token")
assert validate_csrf_token(request, token) is True
assert validate_csrf_token(request, "not-the-token") is False
def test_csrf_token_is_stable_across_renders_in_one_session(
app_and_templates, make_request
):
"""A form rendered on one page must still validate after the user
loads another page in the same session."""
_, templates = app_and_templates
session: dict = {}
first = _csrf_from(
_render(templates, request=make_request(session), name="base.html")
)
second = _csrf_from(
_render(templates, request=make_request(session), name="base.html")
)
assert first == second
def test_explicit_csrf_token_in_context_is_not_clobbered(
app_and_templates, make_request
):
"""A route that passes its own csrf_token keeps it."""
_, templates = app_and_templates
html = _render(
templates,
make_request({}),
"base.html",
{"csrf_token": lambda: "route-supplied-token"},
)
assert _csrf_from(html) == "route-supplied-token"
# ---------------------------------------------------------------------------
# 5. Flash messages across a redirect
# ---------------------------------------------------------------------------
def test_flash_survives_a_redirect_and_is_consumed_once(app_and_templates):
"""Flask's flash()/get_flashed_messages() contract: set on request A,
read on the request the redirect lands on, gone on request C. Driven
through a real SessionMiddleware + cookie jar so the session
round-trip is exercised, not simulated."""
from fastapi import FastAPI
from starlette.middleware.sessions import SessionMiddleware
from starlette.responses import RedirectResponse
from starlette.testclient import TestClient
from local_deep_research.web.dependencies.flash import flash
_, templates = app_and_templates
message = "Your password has been changed."
app = FastAPI()
app.add_middleware(SessionMiddleware, secret_key="census-secret-key")
@app.get("/flash-then-redirect")
def _set(request: Request):
flash(request, message, "success")
return RedirectResponse("/render", status_code=303)
@app.get("/render")
def _render_page(request: Request):
return templates.TemplateResponse(
request=request, name="auth/login.html", context={}
)
client = TestClient(app)
landed = client.get("/flash-then-redirect", follow_redirects=True)
assert [r.status_code for r in landed.history] == [303]
assert message in landed.text, "flash lost across the redirect"
# Category must survive too: login.html picks the alert class from it.
assert "alert-success" in landed.text
again = client.get("/render")
assert again.status_code == 200
assert message not in again.text, "flash was not consumed on read"
@pytest.mark.parametrize(
("category", "alert_class"),
[
("success", "alert-success"),
("info", "alert-info"),
# login.html routes every other category through the else-branch.
("error", "alert-warning"),
],
)
def test_flash_category_selects_the_alert_class(
app_and_templates, category, alert_class
):
"""with_categories=true must yield (category, message) pairs. If the
shim dropped categories -- returning bare strings -- login.html's
`{% for category, message in messages %}` unpacks the characters of
the string instead, and the alert class is chosen from garbage."""
from fastapi import FastAPI
from starlette.middleware.sessions import SessionMiddleware
from starlette.testclient import TestClient
from local_deep_research.web.dependencies.flash import flash
_, templates = app_and_templates
app = FastAPI()
app.add_middleware(SessionMiddleware, secret_key="census-secret-key")
@app.get("/flash")
def _page(request: Request):
flash(request, f"msg-{category}", category)
return templates.TemplateResponse(
request=request, name="auth/login.html", context={}
)
html = TestClient(app).get("/flash").text
assert f"msg-{category}" in html
assert f'class="alert {alert_class} alert-dismissible"' in html
# ---------------------------------------------------------------------------
# 6. Default context completeness (Flask's render_template_with_defaults
# + inject_frontend_constants)
# ---------------------------------------------------------------------------
def test_version_is_injected_into_every_render(app_and_templates, make_request):
"""main passed version=__version__ on every render; the sidebar badge
links to the matching release tag. Without injection the badge is
empty and points at .../releases/tag/v."""
from local_deep_research.__version__ import __version__
_, templates = app_and_templates
html = _render(templates, make_request({}), "base.html", {})
assert f"/releases/tag/v{__version__}" in html, (
"sidebar version link lost the injected version"
)
assert '/releases/tag/v"' not in html
def test_has_encryption_is_injected_into_every_render(
app_and_templates, make_request
):
"""settings_dashboard.html gates its 'encryption unavailable' warning
on `{% if not has_encryption %}`; an Undefined value is falsy, so a
dropped injection shows the warning on encrypted installs."""
from local_deep_research.database.encrypted_db import db_manager
_, templates = app_and_templates
html = _render(templates, make_request({}), "settings_dashboard.html", {})
warning = "Database encryption is not available"
assert (warning in html) is (not db_manager.has_encryption)
def test_research_status_enum_context_covers_every_enum_member(
app_and_templates, make_request
):
"""base.html publishes ResearchStatus to JS. A member missing from
the injected dict is a silent frontend state bug."""
from local_deep_research.constants import ResearchStatus
_, templates = app_and_templates
html = _render(templates, make_request({}), "base.html", {})
match = re.search(
r"window\.RESEARCH_STATUS = Object\.freeze\((\{.*?\})\);", html
)
assert match is not None, "base.html lost window.RESEARCH_STATUS"
published = json.loads(match.group(1))
assert published == {m.name: m.value for m in ResearchStatus}
terminal = re.search(
r"window\.RESEARCH_TERMINAL_STATES = "
r"Object\.freeze\(new Set\((\[.*?\])\)\);",
html,
)
assert terminal is not None, "base.html lost RESEARCH_TERMINAL_STATES"
states = json.loads(terminal.group(1))
assert states, "terminal state list rendered empty"
assert set(states) <= set(published.values())
def test_log_limits_context_matches_the_constants(
app_and_templates, make_request
):
from local_deep_research.constants import (
HISTORY_LOGS_DEFAULT_LIMIT,
HISTORY_LOGS_HARD_CAP,
)
_, templates = app_and_templates
html = _render(templates, make_request({}), "base.html", {})
match = re.search(
r"window\.LDR_LOG_LIMITS = Object\.freeze\((\{.*?\})\);", html
)
assert match is not None, "base.html lost window.LDR_LOG_LIMITS"
assert json.loads(match.group(1)) == {
"default": HISTORY_LOGS_DEFAULT_LIMIT,
"hard_cap": HISTORY_LOGS_HARD_CAP,
}
def _body_scope(html: str) -> str:
match = re.search(r'<body data-scope="([^"]*)"', html)
assert match is not None, "base.html lost <body data-scope=…>"
return match.group(1)
def test_egress_scope_defaults_for_an_anonymous_session(
app_and_templates, make_request
):
from local_deep_research.security.egress.policy import (
DEFAULT_EGRESS_SCOPE,
)
_, templates = app_and_templates
html = _render(templates, make_request({}), "base.html", {})
assert _body_scope(html) == DEFAULT_EGRESS_SCOPE
@pytest.mark.parametrize(
("stored", "expected"),
[
("STRICT", "strict"),
(" Public_Only ", "public_only"),
("retired-value", "adaptive"),
],
)
def test_stored_egress_scope_is_canonicalised_before_it_reaches_the_body(
app_and_templates, make_request, monkeypatch, stored, expected
):
"""Regression for the helper restored during this migration.
base.html's CSS selectors are `body[data-scope="strict"]` -- attribute
values are case-sensitive, so an un-canonicalised "STRICT" silently
kills the strict-scope visual cue."""
import local_deep_research.database.session_context as session_context
import local_deep_research.utilities.db_utils as db_utils
_, templates = app_and_templates
class _SettingsManager:
def get_setting(self, key, default=None):
return stored if key == "policy.egress_scope" else default
@contextlib.contextmanager
def _fake_session(_username):
yield object()
monkeypatch.setattr(
db_utils, "get_settings_manager", lambda *a, **k: _SettingsManager()
)
monkeypatch.setattr(session_context, "get_user_db_session", _fake_session)
html = _render(
templates, make_request({"username": "scoped"}), "base.html", {}
)
assert _body_scope(html) == expected
# ---------------------------------------------------------------------------
# 7. Both render entry points must agree
# ---------------------------------------------------------------------------
def test_render_template_helper_and_templateresponse_inject_the_same_defaults(
app_and_templates, make_request
):
"""Routes use two entry points: dependencies.template_helpers
.render_template and templates.TemplateResponse directly. Anything
only one of them injects is a page that differs by which helper its
route happened to pick -- exactly how egress_scope regressed."""
from local_deep_research.web.dependencies.template_helpers import (
render_template,
)
_, templates = app_and_templates
session = {"username": "parity"}
direct = _render(
templates,
make_request(dict(session)),
"base.html",
{"active_page": "research"},
)
via_helper = render_template(
make_request(dict(session)),
"base.html",
{"active_page": "research"},
).body.decode("utf-8")
# The two calls mint tokens from two independent session dicts, so
# normalise every 64-hex CSRF token (meta tag and hidden inputs) out
# before comparing. Response authorizations also differ; their shape,
# isolation and header agreement are checked in test_template_security_policy.
# Everything else must match byte for byte.
strip = re.compile(r"\b[0-9a-f]{64}\b")
authorization = re.compile(
r'(?P<name>nonce|data-template-token|data-ldr-template-token)="[A-Za-z0-9_-]{43}"'
)
def normalized(html):
return authorization.sub(
lambda match: f'{match["name"]}="RESPONSE"',
strip.sub("TOKEN", html),
)
assert normalized(direct) == normalized(via_helper)
assert (
len(authorization.findall(direct))
== len(authorization.findall(via_helper))
> 0
)
assert len(strip.findall(direct)) == len(strip.findall(via_helper)) > 0
def test_render_template_helper_honours_status_code(
app_and_templates, make_request
):
from local_deep_research.web.dependencies.template_helpers import (
render_template,
)
_, templates = app_and_templates
response = render_template(
make_request({}), "auth/login.html", {}, status_code=401
)
assert response.status_code == 401
assert response.headers["content-type"].startswith("text/html")
# ---------------------------------------------------------------------------
# 8. Known defect (latent): positional-context calls lose the defaults
# ---------------------------------------------------------------------------
def test_positional_context_call_still_injects_defaults(
app_and_templates, make_request
):
from local_deep_research.__version__ import __version__
_, templates = app_and_templates
response = templates.TemplateResponse(
make_request({"username": "positional"}),
"base.html",
{"active_page": "research"},
)
assert f"/releases/tag/v{__version__}" in response.body.decode("utf-8")
def test_positional_status_code_is_kept(app_and_templates, make_request):
"""Every positional parameter keeps its meaning, not only the context."""
_, templates = app_and_templates
response = templates.TemplateResponse(
make_request({"username": "positional"}),
"base.html",
{"active_page": "research"},
404,
)
assert response.status_code == 404
def test_context_given_twice_is_a_type_error(app_and_templates, make_request):
"""Positional and keyword context together is a caller bug, as in Python."""
_, templates = app_and_templates
with pytest.raises(
TypeError, match="multiple values for argument 'context'"
):
templates.TemplateResponse(
make_request({"username": "positional"}),
"base.html",
{"active_page": "research"},
context={"active_page": "history"},
)