~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Comprehensive tests for PDFService.
Tests PDF generation, markdown conversion, CSS handling, and metadata.
"""
import http.client
import io
import subprocess
import zlib
import pytest
from unittest.mock import patch
def _host_has_cjk_fonts() -> bool:
"""True if fontconfig reports any CJK-capable family on the host.
Used to gate the strong glyph-embedding assertion in
test_handles_cjk_content_embeds_glyphs: Docker CI (where this PR
installs fonts-noto-cjk) and properly-configured dev hosts run the
assertion; bare pip/macOS/Windows installs without Noto CJK skip it.
"""
try:
result = subprocess.run(
["fc-list", ":lang=zh"],
capture_output=True,
text=True,
timeout=5,
)
except (FileNotFoundError, subprocess.TimeoutExpired):
return False
return bool(result.stdout.strip())
class TestPDFServiceInit:
"""Tests for PDFService initialization."""
def test_init_creates_minimal_css(self):
"""Test that initialization creates minimal CSS."""
from local_deep_research.web.services.pdf_service import PDFService
service = PDFService()
assert service.minimal_css is not None
def test_minimal_css_includes_page_settings(self):
"""Test that minimal CSS includes @page settings."""
from local_deep_research.web.services.pdf_service import PDFService
service = PDFService()
# The CSS object wraps the string, so we check it was created
assert service.minimal_css is not None
class TestPDFServiceMarkdownToHtml:
"""Tests for _markdown_to_html method."""
@pytest.fixture
def service(self):
"""Create PDFService instance."""
from local_deep_research.web.services.pdf_service import PDFService
return PDFService()
def test_converts_basic_markdown(self, service, simple_markdown):
"""Test conversion of basic markdown to HTML."""
html = service._markdown_to_html(simple_markdown)
# Markdown adds id attribute to headings
assert "<h1" in html and "Hello World</h1>" in html
assert "<p>This is a test.</p>" in html
def test_includes_doctype_and_structure(self, service, simple_markdown):
"""Test that HTML includes proper structure."""
html = service._markdown_to_html(simple_markdown)
assert "<!DOCTYPE html>" in html
assert "<html>" in html
assert "</html>" in html
assert "<head>" in html
assert "</head>" in html
assert "<body>" in html
assert "</body>" in html
def test_includes_charset_meta(self, service, simple_markdown):
"""Test that HTML includes UTF-8 charset."""
html = service._markdown_to_html(simple_markdown)
assert '<meta charset="utf-8">' in html
def test_includes_title_when_provided(self, service, simple_markdown):
"""Test that title is included when provided."""
html = service._markdown_to_html(simple_markdown, title="Test Title")
assert "<title>Test Title</title>" in html
def test_excludes_title_when_not_provided(self, service, simple_markdown):
"""Test that no title tag when title not provided."""
html = service._markdown_to_html(simple_markdown)
assert "<title>" not in html
def test_includes_metadata_as_meta_tags(
self, service, simple_markdown, sample_metadata
):
"""Test that metadata is included as meta tags."""
html = service._markdown_to_html(
simple_markdown, metadata=sample_metadata
)
assert '<meta name="author" content="Test Author">' in html
assert '<meta name="date" content="2024-01-15">' in html
assert '<meta name="subject" content="Test Subject">' in html
def test_includes_ldr_attribution_footer(self, service, simple_markdown):
"""Test that LDR attribution footer is included."""
html = service._markdown_to_html(simple_markdown)
assert "Generated by" in html
assert "Local Deep Research" in html
def test_converts_tables(self, service, sample_markdown):
"""Test that markdown tables are converted."""
html = service._markdown_to_html(sample_markdown)
assert "<table>" in html
assert "<th>" in html
assert "Column 1" in html
def test_converts_code_blocks(self, service, sample_markdown):
"""Test that fenced code blocks are converted."""
html = service._markdown_to_html(sample_markdown)
assert "<code" in html or "<pre>" in html
assert "def example():" in html
def test_converts_lists(self, service, sample_markdown):
"""Test that lists are converted."""
html = service._markdown_to_html(sample_markdown)
assert "<li>" in html
assert "Item 1" in html
def test_converts_bold_and_italic(self, service, sample_markdown):
"""Test that bold and italic text are converted."""
html = service._markdown_to_html(sample_markdown)
assert "<strong>bold</strong>" in html
assert "<em>italic</em>" in html
def test_handles_special_characters(
self, service, markdown_with_special_chars
):
"""Test handling of special characters."""
html = service._markdown_to_html(markdown_with_special_chars)
# Unicode should pass through
assert "café" in html
assert "naïve" in html
assert "日本語" in html
class TestPDFServiceMarkdownToPdf:
"""Tests for markdown_to_pdf method."""
@pytest.fixture
def service(self):
"""Create PDFService instance."""
from local_deep_research.web.services.pdf_service import PDFService
return PDFService()
def test_returns_bytes(self, service, simple_markdown):
"""Test that markdown_to_pdf returns bytes."""
result = service.markdown_to_pdf(simple_markdown)
assert isinstance(result, bytes)
def test_returns_valid_pdf(self, service, simple_markdown):
"""Test that result is a valid PDF (starts with %PDF)."""
result = service.markdown_to_pdf(simple_markdown)
assert result.startswith(b"%PDF")
def test_pdf_has_reasonable_size(self, service, simple_markdown):
"""Test that generated PDF has a reasonable size."""
result = service.markdown_to_pdf(simple_markdown)
# A simple PDF should be at least a few KB
assert len(result) > 1000
def test_applies_title(self, service, simple_markdown):
"""Test that title is applied to PDF."""
result = service.markdown_to_pdf(simple_markdown, title="My Title")
# PDF was generated successfully
assert result.startswith(b"%PDF")
assert len(result) > 0
def test_applies_metadata(self, service, simple_markdown, sample_metadata):
"""Test that metadata is applied to PDF."""
result = service.markdown_to_pdf(
simple_markdown, metadata=sample_metadata
)
assert result.startswith(b"%PDF")
def test_applies_custom_css(self, service, simple_markdown, custom_css):
"""Test that custom CSS is applied."""
result = service.markdown_to_pdf(simple_markdown, custom_css=custom_css)
assert result.startswith(b"%PDF")
def test_uses_minimal_css_when_no_custom_css(
self, service, simple_markdown
):
"""Test that minimal CSS is used by default."""
result = service.markdown_to_pdf(simple_markdown)
assert result.startswith(b"%PDF")
def test_handles_empty_markdown(self, service):
"""Test handling of empty markdown."""
result = service.markdown_to_pdf("")
assert result.startswith(b"%PDF")
def test_handles_large_markdown(self, service):
"""Test handling of large markdown content."""
# Create a large markdown document
large_content = "# Large Document\n\n"
large_content += ("This is a paragraph. " * 100 + "\n\n") * 50
result = service.markdown_to_pdf(large_content)
assert result.startswith(b"%PDF")
# Large content should produce larger PDF
assert len(result) > 10000
def test_handles_markdown_with_all_features(self, service, sample_markdown):
"""Test handling of markdown with tables, code, lists."""
result = service.markdown_to_pdf(sample_markdown)
assert result.startswith(b"%PDF")
def test_handles_cjk_content(self, service):
"""Regression guard for #4055 — CJK text must not crash export.
Glyph visibility depends on the host having CJK fonts installed
(e.g. fonts-noto-cjk). This test pins the no-crash contract; the
glyph-embedding contract is checked separately in
test_handles_cjk_content_embeds_glyphs.
"""
cjk_markdown = (
"# 中文标题\n\n这是一个测试。\n\n"
"## 日本語の見出し\n\nテスト本文。\n\n"
"## 한국어 제목\n\n본문 테스트.\n"
)
result = service.markdown_to_pdf(cjk_markdown)
assert result.startswith(b"%PDF")
assert len(result) > 1000
@pytest.mark.skipif(
not _host_has_cjk_fonts(),
reason="no CJK fonts on host; install fonts-noto-cjk to run",
)
def test_handles_cjk_content_embeds_glyphs(self, service):
"""End-to-end glyph check for #4055.
Round-trips CJK text through markdown → PDF → text extraction.
If no CJK-capable font is available to WeasyPrint, glyphs are
dropped and pypdf's extract_text returns the rendered text
without them.
What this catches: removal of fonts-noto-cjk from the runtime
Docker image (the load-bearing half of the fix) — pytest-tests
runs inside that image, so the test fails when fonts are gone.
What this does NOT catch: removal of the CSS CJK fallback list.
On Linux, fontconfig auto-substitutes a glyph-bearing family
even for a Latin-only CSS stack, so the assertion still passes.
The CSS fallbacks matter on Windows/macOS, which CI doesn't run.
"""
import pypdf
cjk_markdown = (
"# 中文标题\n\n这是一个测试。\n\n"
"## 日本語の見出し\n\nテスト本文。\n\n"
"## 한국어 제목\n\n본문 테스트.\n"
)
pdf_bytes = service.markdown_to_pdf(cjk_markdown)
reader = pypdf.PdfReader(io.BytesIO(pdf_bytes))
extracted = "".join(page.extract_text() for page in reader.pages)
for phrase in (
"中文标题",
"这是一个测试",
"日本語",
"テスト",
"한국어",
"본문",
):
assert phrase in extracted, (
f"CJK glyphs missing from PDF: {phrase!r} not found in "
f"extracted text — WeasyPrint could not match a glyph "
f"(check that fonts-noto-cjk is installed on the host)"
)
def test_logs_pdf_size(self, service, simple_markdown):
"""Test that PDF size is logged."""
with patch(
"local_deep_research.web.services.pdf_service.logger"
) as mock_logger:
service.markdown_to_pdf(simple_markdown)
mock_logger.info.assert_called_once()
call_args = mock_logger.info.call_args[0][0]
assert "Generated PDF" in call_args
assert "bytes" in call_args
class TestPDFServiceErrorHandling:
"""Tests for error handling in PDFService."""
@pytest.fixture
def service(self):
"""Create PDFService instance."""
from local_deep_research.web.services.pdf_service import PDFService
return PDFService()
def test_raises_exception_on_html_error(self, service):
"""Test that exceptions are raised on HTML conversion errors."""
with patch.object(
service, "_markdown_to_html", side_effect=Exception("HTML error")
):
with pytest.raises(Exception) as exc_info:
service.markdown_to_pdf("test")
assert "HTML error" in str(exc_info.value)
def test_logs_exception_on_error(self, service):
"""Test that exceptions are logged."""
with patch.object(
service, "_markdown_to_html", side_effect=Exception("Test error")
):
with patch(
"local_deep_research.web.services.pdf_service.logger"
) as mock_logger:
with pytest.raises(Exception):
service.markdown_to_pdf("test")
mock_logger.exception.assert_called_once()
class TestPDFServiceSingleton:
"""Tests for get_pdf_service singleton."""
def test_get_pdf_service_returns_instance(self):
"""Test that get_pdf_service returns a PDFService instance."""
from local_deep_research.web.services.pdf_service import (
get_pdf_service,
PDFService,
)
service = get_pdf_service()
assert isinstance(service, PDFService)
def test_get_pdf_service_returns_same_instance(self):
"""Test that get_pdf_service returns the same instance."""
from local_deep_research.web.services.pdf_service import (
get_pdf_service,
)
service1 = get_pdf_service()
service2 = get_pdf_service()
assert service1 is service2
def test_singleton_can_be_reset(self):
"""Test that singleton can be reset for testing."""
import local_deep_research.web.services.pdf_service as pdf_module
# Reset singleton
pdf_module._pdf_service = None
service1 = pdf_module.get_pdf_service()
# Reset again
pdf_module._pdf_service = None
service2 = pdf_module.get_pdf_service()
# Different instances after reset
assert service1 is not service2
class TestRefusedResourceDoesNotAbortTheExport:
"""A refused fetch must be skipped, whichever at-rule asked for it (#6466).
WeasyPrint does not treat them alike. An `@import` it cannot fetch is
skipped and the render completes; an `@color-profile` it cannot fetch is
not, because the `fetch()` call in `weasyprint/css/__init__.py` sits
outside the `try`, so the `URLFetchingError` escapes `CSS.__init__` and
then `markdown_to_pdf`, and the export route answers a generic 500.
Raw HTML in report text is escaped by `_markdown_to_html`, so a `<style>`
element cannot come from report markdown, and the export route passes no
custom stylesheet; the retry in `_render_pdf` is defence in depth for a
stylesheet that does reach the document. These tests therefore inject the
`<style>` at the HTML layer (by stubbing `_markdown_to_html`), which is
where WeasyPrint parses it at render time. `validate_url` and `HTML` are
wrapped, not replaced: the real guard still decides, and the wrappers
record which URLs it was asked about and which fetcher each render pass
used. The URLs used below are the cloud metadata address the SSRF guard
always blocks, so the refusal comes from this service's own fetcher.
"""
BLOCKED = "http://169.254.169.254/q.icc"
BLOCKED_IMPORT = "http://169.254.169.254/x.css"
def _export(self, stylesheet: str):
"""Export a report whose document carries ``stylesheet`` in a
``<style>`` element; return the PDF, the URLs the guard was asked
about, and the url_fetcher of each render pass."""
from local_deep_research.web.services import pdf_service
# Settle the lazy import first, or its first run would replace the
# `HTML` wrapper patched in below with WeasyPrint's own class.
pdf_service._ensure_weasyprint()
html_doc = (
"<!DOCTYPE html><html><head><meta charset='utf-8'>"
f"<style>{stylesheet}</style>"
"</head><body><h1>Report</h1><p>Body text.</p></body></html>"
)
asked: list[str] = []
real_validate_url = pdf_service.validate_url
def recording_validate_url(url, *args, **kwargs):
asked.append(url)
return real_validate_url(url, *args, **kwargs)
passes: list = []
original_html = pdf_service.HTML
def recording_html(*args, **kwargs):
passes.append(kwargs.get("url_fetcher"))
return original_html(*args, **kwargs)
with (
patch.object(pdf_service, "validate_url", recording_validate_url),
patch.object(pdf_service, "HTML", recording_html),
patch.object(
pdf_service.PDFService,
"_markdown_to_html",
lambda self, content, title=None, metadata=None: html_doc,
),
):
pdf = pdf_service.PDFService().markdown_to_pdf(
"# Report\n\nBody text.\n", title="t"
)
return pdf, asked, passes
def test_a_refused_color_profile_still_exports(self):
"""Without the retry the refused `@color-profile` aborts the render
and `markdown_to_pdf` raises, so this fails if the retry is removed."""
from local_deep_research.web.services import pdf_service
pdf, asked, passes = self._export(
"@color-profile --q { src: url('"
+ self.BLOCKED
+ "'); components: 3; }"
)
assert pdf.startswith(b"%PDF")
# The guard refused the profile on the first pass, which that refusal
# aborted, and again on the retry, which skipped it.
assert asked == [self.BLOCKED, self.BLOCKED], asked
assert len(passes) == 2, passes
assert passes[0] is pdf_service._safe_url_fetcher
assert isinstance(passes[1], pdf_service._SkippingUrlFetcher)
def test_a_refused_import_still_exports(self):
"""The control, and the reason the report calls this an asymmetry: this
shape already worked, so a fix that made both fail would satisfy a
weaker test than this one. WeasyPrint skips the refused `@import`
itself, so the export completes in one pass, without the retry."""
from local_deep_research.web.services import pdf_service
pdf, asked, passes = self._export(
f"@import url('{self.BLOCKED_IMPORT}');"
)
assert pdf.startswith(b"%PDF")
# The stylesheet reached the renderer and the guard refused the import.
assert asked == [self.BLOCKED_IMPORT], asked
assert passes == [pdf_service._safe_url_fetcher], passes
def test_the_ordinary_report_is_rendered_once(self):
"""Second control. The retry must be reached only by the render that
would otherwise have failed: a report with no external resource must
not pay for a second pass, and must not go through the fetcher that
turns refusals into empty resources.
"""
from local_deep_research.web.services import pdf_service
# Settle the lazy import first, as `_export` does: captured before
# it, `original` would be None, the import inside the call would
# replace the wrapper, and leaving the patch would restore None for
# every later test in this worker.
pdf_service._ensure_weasyprint()
calls = []
original = pdf_service.HTML
def counting_html(*args, **kwargs):
calls.append(kwargs.get("url_fetcher"))
return original(*args, **kwargs)
with patch.object(pdf_service, "HTML", counting_html):
pdf = pdf_service.PDFService().markdown_to_pdf(
"# Report\n\nBody text.\n", title="t"
)
assert pdf.startswith(b"%PDF")
assert calls == [pdf_service._safe_url_fetcher]
def test_the_skipping_fetcher_still_refuses_to_fetch_a_blocked_url(self):
"""The security property, asserted rather than assumed. The retry
changes what WeasyPrint is TOLD about a refused URL, not whether the
request is made: `validate_url` still runs first and still decides, and
an empty body is returned without touching the network.
"""
from local_deep_research.web.services import pdf_service
# Settle the lazy import first: which of the two shapes below comes
# back depends on it, and leaving that to whichever test ran earlier
# made this cell pass alone and fail in the file.
pdf_service._ensure_weasyprint()
# A fetcher that would happily serve the blocked URL, so the assertion
# is that it was never ASKED. Patching it to None instead proved
# nothing: a fetcher that bypassed the guard would raise on the None
# and be swallowed by the same `except`, and the cell stayed green.
fetched: list[str] = []
class _WouldServeAnything:
def fetch(self, url):
fetched.append(url)
return {"string": b"metadata", "mime_type": "text/plain"}
with patch.object(pdf_service, "_URL_FETCHER", _WouldServeAnything()):
resource = pdf_service._SkippingUrlFetcher()(self.BLOCKED)
assert fetched == [], "the guard must decide before any request is made"
# WeasyPrint 68 returns a response read through `read()`; older
# versions return a dict. Either way the body is empty and nothing was
# fetched.
body = (
resource["string"]
if isinstance(resource, dict)
else resource.read()
)
assert body == b""
def test_an_allowed_url_is_fetched_normally_by_the_skipping_fetcher(self):
"""And the accept control for it: the retry fetcher is not a blanket
"return nothing", or the retried render would silently drop every
legitimate resource the report references.
``validate_url`` is patched (not just ``_URL_FETCHER``) so the cell
performs no network I/O: the real validator resolves hostnames via
``socket.getaddrinfo``, and an offline/sandboxed runner used to
turn that lookup into a misleading ``is payload`` failure through
the fetcher's blanket except (#6589). WeasyPrint is loaded first:
a cold ``_ensure_weasyprint`` inside the call would replace the
patched ``_URL_FETCHER`` with the real one, so the cell only passed
when an earlier test had loaded it. The fetch is wrapped in the DNS
pin, which resolves the host once through the resolver seam
``dns_pinning._real_getaddrinfo``; that seam is patched to a fixed
TEST-NET-1 answer (192.0.2.10, which passes the SSRF policy but is
not routable), so no real lookup happens. ``socket.getaddrinfo``
itself is left alone: replacing it would remove the pin shim, and
the fetcher refuses to fetch without it.
"""
import socket
from local_deep_research.security import dns_pinning
from local_deep_research.web.services import pdf_service
payload = {"string": b"body", "mime_type": "text/css"}
class _Fetcher:
def fetch(self, url):
return payload
pdf_service._ensure_weasyprint() # populate the lazy _URL_FETCHER
def _public(host, port, *_args, **_kwargs):
return [
(
socket.AF_INET,
socket.SOCK_STREAM,
socket.IPPROTO_TCP,
"",
("192.0.2.10", port or 0),
)
]
with (
patch.object(pdf_service, "validate_url", return_value=True),
patch.object(pdf_service, "_URL_FETCHER", _Fetcher()),
patch.object(dns_pinning, "_real_getaddrinfo", _public),
):
assert (
pdf_service._SkippingUrlFetcher()("https://example.com/x.css")
is payload
), "an allowed URL must reach the (patched) module URL fetcher"
class TestSkippingFetcherHardening:
"""Pins for `_SkippingUrlFetcher`, the fetcher the retry in
`_render_pdf` renders with (#6588)."""
# userinfo, path and query all carry secrets the logs must never see;
# the host is the metadata address the SSRF guard always refuses.
CREDENTIALED_BLOCKED = (
"http://admin:[email protected]/latest/meta.icc?token=TOK123"
)
@staticmethod
def _capture_pdf_service_logs(messages):
from loguru import logger as loguru_logger
def sink(message):
# The formatted line, traceback included: a `logger.exception`
# would carry the exception's message (which can embed the raw
# URL) into the log through it.
record = message.record
if record["name"].startswith(
"local_deep_research.web.services.pdf_service"
):
messages.append(str(message))
return loguru_logger.add(sink, level="DEBUG")
def test_refusal_reason_reaches_the_log(self):
"""The skip log must say WHY the resource was refused -- by exception
type, with the URL redacted and no credential text anywhere."""
from loguru import logger as loguru_logger
from local_deep_research.web.services import pdf_service
messages: list[str] = []
handler = self._capture_pdf_service_logs(messages)
loguru_logger.enable("local_deep_research")
fetcher = pdf_service._SkippingUrlFetcher()
try:
result = fetcher(self.CREDENTIALED_BLOCKED)
finally:
loguru_logger.disable("local_deep_research")
loguru_logger.remove(handler)
assert result is not None
assert fetcher.skipped == [self.CREDENTIALED_BLOCKED]
skip_lines = [
m for m in messages if "Skipping unavailable resource" in m
]
assert len(skip_lines) == 1, messages
assert "UnsafePDFResourceURLError" in skip_lines[0]
assert "http://169.254.169.254" in skip_lines[0]
joined = "\n".join(messages)
for secret in ("S3CRETPW", "admin", "TOK123", "meta.icc"):
assert secret not in joined, joined
@pytest.mark.parametrize(
"error",
[
# What a hostile server can make http.client / the decoder raise:
# none of these is an OSError or ValueError.
http.client.BadStatusLine("GARBAGE"),
http.client.LineTooLong("header line"),
http.client.HTTPException("got more than 100 headers"),
zlib.error("invalid stored block lengths"),
TypeError("bad glue"),
],
ids=lambda e: type(e).__name__,
)
def test_any_fetch_failure_is_skipped_not_raised(self, error):
"""The retry exists to skip whatever the first pass could not survive,
so nothing a fetch raises may escape it: an escape aborts the retried
render (and on WeasyPrint 70 its fetch() then reads
`_fail_on_errors` off the fetcher) and a completed export becomes a
500. A programming error is not hidden -- its type is logged -- but
its message is not, since `InvalidURL` and friends embed the raw URL.
"""
from loguru import logger as loguru_logger
from local_deep_research.web.services import pdf_service
error.args = error.args + ("S3CRETPW",)
messages: list[str] = []
handler = self._capture_pdf_service_logs(messages)
loguru_logger.enable("local_deep_research")
fetcher = pdf_service._SkippingUrlFetcher()
try:
with patch.object(
pdf_service, "_safe_url_fetcher", side_effect=error
):
result = fetcher("https://example.com/x.css")
finally:
loguru_logger.disable("local_deep_research")
loguru_logger.remove(handler)
assert result is not None
assert fetcher.skipped == ["https://example.com/x.css"]
joined = "\n".join(messages)
assert type(error).__name__ in joined
assert "S3CRETPW" not in joined, joined
def test_the_empty_resource_survives_a_constructor_change(self):
"""WeasyPrint 69+ asserts the fetcher returned a `URLFetcherResponse`
-- a dict is rejected -- so when the full construction is refused
(an upstream signature change) the fallback must still be one. The
warning naming that failure must log the exception type only: an
arbitrary upstream exception message is not vetted and could echo
secrets embedded in headers/body by a caller."""
from loguru import logger as loguru_logger
from local_deep_research.web.services import pdf_service
pdf_service._ensure_weasyprint()
real = pdf_service.URLFetcherResponse
if real is None:
pytest.skip("WeasyPrint < 68 has no URLFetcherResponse")
class _ChangedSignature(real):
def __init__(self, *args, **kwargs):
if "body" in kwargs:
raise TypeError(
"unexpected keyword argument 'body' S3CRETPW"
)
super().__init__(*args, **kwargs)
messages: list[str] = []
handler = self._capture_pdf_service_logs(messages)
loguru_logger.enable("local_deep_research")
try:
with patch.object(
pdf_service, "URLFetcherResponse", _ChangedSignature
):
resource = pdf_service._empty_resource_for("https://x.test/a")
finally:
loguru_logger.disable("local_deep_research")
loguru_logger.remove(handler)
assert isinstance(resource, real)
assert resource.read() == b""
joined = "\n".join(messages)
assert "TypeError" in joined, joined
assert "S3CRETPW" not in joined, joined
def test_the_fetcher_carries_the_fail_on_errors_contract(self):
"""WeasyPrint 70's fetch() reads `_fail_on_errors` off the fetcher
when it catches from it; a missing attribute is an AttributeError
mid-render."""
from local_deep_research.web.services import pdf_service
assert (
getattr(pdf_service._SkippingUrlFetcher(), "_fail_on_errors", None)
is False
)
class TestRetriedRenderEndToEnd:
"""The retry as `_render_pdf` actually runs it, through real WeasyPrint.
The render is forced into its retry by a refused `@color-profile` (the
at-rule whose fetch failure aborts the render). `_URL_FETCHER` is
replaced by a spy, so nothing leaves the process: the assertions are on
what the spy was *asked* to fetch.
Report markdown now escapes raw HTML, so the stylesheet can no longer
arrive from report text; it is injected at the HTML layer (by stubbing
`_markdown_to_html`), which is where WeasyPrint parses it at render time
and where `_render_pdf`'s retry lives. The retry and the SSRF guard are
thus exercised independently of the markdown-escaping step.
"""
CREDENTIALED_BLOCKED = (
"http://admin:[email protected]/latest/meta.icc?token=TOK123"
)
HOSTILE = "https://attacker.example/bad.icc"
def _render(self):
import socket
from loguru import logger as loguru_logger
from local_deep_research.security import dns_pinning
from local_deep_research.web.services import pdf_service
pdf_service._ensure_weasyprint()
fetched: list[str] = []
response_type = pdf_service.URLFetcherResponse
class _SpyFetcher:
def fetch(self, url):
fetched.append(url)
if url == TestRetriedRenderEndToEnd.HOSTILE:
# A server answering with a garbage status line.
raise http.client.BadStatusLine("GARBAGE")
if response_type is not None:
return response_type(url, body=b"")
return {"string": b"", "mime_type": "text/plain"}
def fake_validate_url(url, *args, **kwargs):
# The metadata host is refused, as the real guard always does;
# the stand-in public host is allowed without a DNS lookup.
return "169.254.169.254" not in url
def public_resolver(host, port, *args, **kwargs):
# The DNS pin around each fetch resolves the host once through
# this seam; a fixed TEST-NET-1 answer (passes the SSRF policy,
# not routable) keeps the cell offline.
return [
(
socket.AF_INET,
socket.SOCK_STREAM,
socket.IPPROTO_TCP,
"",
("192.0.2.10", port or 0),
)
]
# The hostile resource comes first, so it is what aborts the first
# pass; the refused one is then reached only by the retry.
html_doc = (
"<!DOCTYPE html><html><head><meta charset='utf-8'>"
"<style>"
"@color-profile --a { src: url('"
+ self.HOSTILE
+ "'); components: 3; }"
"@color-profile --b { src: url('"
+ self.CREDENTIALED_BLOCKED
+ "'); components: 3; }"
"</style>"
"</head><body><h1>Report</h1><p>Body text.</p></body></html>"
)
html_calls: list = []
original_html = pdf_service.HTML
def recording_html(*args, **kwargs):
html_calls.append(kwargs.get("url_fetcher"))
return original_html(*args, **kwargs)
messages: list[str] = []
handler = TestSkippingFetcherHardening._capture_pdf_service_logs(
messages
)
loguru_logger.enable("local_deep_research")
try:
with (
patch.object(pdf_service, "_URL_FETCHER", _SpyFetcher()),
patch.object(pdf_service, "validate_url", fake_validate_url),
patch.object(dns_pinning, "_real_getaddrinfo", public_resolver),
patch.object(pdf_service, "HTML", recording_html),
patch.object(
pdf_service.PDFService,
"_markdown_to_html",
lambda self, content, title=None, metadata=None: html_doc,
),
):
pdf = pdf_service.PDFService().markdown_to_pdf(
"# Report\n\nBody text.\n", title="t"
)
finally:
loguru_logger.disable("local_deep_research")
loguru_logger.remove(handler)
return pdf, fetched, messages, html_calls
def test_the_retry_runs_and_still_exports(self):
from local_deep_research.web.services import pdf_service
pdf, _, _, html_calls = self._render()
assert pdf.startswith(b"%PDF")
# First the normal render, then the retry with the skipping fetcher.
assert html_calls[0] is pdf_service._safe_url_fetcher
assert isinstance(html_calls[-1], pdf_service._SkippingUrlFetcher)
def test_a_blocked_url_is_never_fetched_on_the_retry(self):
_, fetched, _, _ = self._render()
assert not any("169.254.169.254" in url for url in fetched), fetched
# The allowed resource did reach the fetcher, so the spy was live.
assert self.HOSTILE in fetched
def test_a_hostile_status_line_on_the_retry_is_skipped(self):
"""`BadStatusLine` is neither an OSError nor a ValueError; raised in
a `@color-profile` fetch on the retry it would abort the retried
render too, unless the retry fetcher skips it."""
pdf, fetched, _, _ = self._render()
assert pdf.startswith(b"%PDF")
# Once by the first pass (whose abort forced the retry), once by the
# retry, which skipped it instead of aborting again.
assert fetched.count(self.HOSTILE) == 2, fetched
def test_the_skipped_summary_lists_only_redacted_urls(self):
_, _, messages, _ = self._render()
summary = [m for m in messages if "skipped resource(s)" in m]
assert len(summary) == 1, messages
line = summary[0]
assert "2 skipped resource(s)" in line
assert "http://169.254.169.254" in line
assert "https://attacker.example" in line
for secret in ("S3CRETPW", "admin", "TOK123", "meta.icc", "bad.icc"):
assert secret not in line, line
# And nowhere else in this module's logs either.
joined = "\n".join(messages)
assert "S3CRETPW" not in joined, joined
def test_each_retry_gets_its_own_skipping_fetcher(self):
"""`_render_pdf` must build a fresh `_SkippingUrlFetcher` per retry.
Nothing else in this module pins that: a module-level shared
instance whose `skipped` list is cleared at the start of each
retry would pass every other test here (each render still ends
up with exactly its own render's skips by the time the summary
is logged), while mixing skip lists across concurrent renders --
one render's retry could clear the list out from under another
render's retry still running against the same instance.
`_render` already records the exact `url_fetcher` object handed
to `HTML(...)` for each pass (`html_calls`); its last entry is
the real `_SkippingUrlFetcher` instance the retry rendered with.
Two full renders' worth of those instances are compared here.
"""
from local_deep_research.web.services import pdf_service
_, _, _, html_calls_one = self._render()
_, _, _, html_calls_two = self._render()
fetcher_one = html_calls_one[-1]
fetcher_two = html_calls_two[-1]
assert isinstance(fetcher_one, pdf_service._SkippingUrlFetcher)
assert isinstance(fetcher_two, pdf_service._SkippingUrlFetcher)
assert fetcher_one is not fetcher_two
assert fetcher_one.skipped is not fetcher_two.skipped
# Each render skips the same two (redacted) URLs; a shared
# instance would show them doubled up on whichever fetcher was
# captured second instead of each holding only its own render's
# skips.
assert len(fetcher_one.skipped) == 2, fetcher_one.skipped
assert len(fetcher_two.skipped) == 2, fetcher_two.skipped