~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
refactor(pdf_service): make WeasyPrint _fail_on_errors contract intrinsic via callable-class fetchers (#6749)
LearningCircuitaba224d5da0b
diff --git a/src/local_deep_research/web/services/pdf_service.py b/src/local_deep_research/web/services/pdf_service.pyindex 1114d29c1f..49c7cb7e50 100644--- a/src/local_deep_research/web/services/pdf_service.py+++ b/src/local_deep_research/web/services/pdf_service.py@@ -117,23 +117,54 @@ URLFetcherResponse = None_EMPTY_RESOURCE_MIME_TYPE = "application/octet-stream"-def _safe_url_fetcher(url):+class _SafeUrlFetcher:"""WeasyPrint url_fetcher that blocks SSRF targets (GHSA-fj2m-qvh9-jq4q)."""- if not validate_url(url):- # A rejected URL is adversarial-shaped and may carry the- # operator's real credentials (RFC 3986 userinfo) or secret- # query tokens; log and raise only scheme://host:port, the- # same discipline every ssrf_validator site follows.- redacted = redact_url_for_log(url)- logger.warning(f"Blocked unsafe URL in PDF rendering: {redacted}")- raise UnsafePDFResourceURLError(- f"Blocked unsafe URL in PDF rendering: {redacted}"- )- _ensure_weasyprint()- return _URL_FETCHER.fetch(url)+ # WeasyPrint 70's fetch() context manager (weasyprint/urls.py) reads+ # ``_fail_on_errors`` off the *url_fetcher itself* after catching an+ # exception it raised — a callable without that attribute turns every+ # blocked URL into an ``AttributeError`` mid-render. Carrying it as a+ # class attribute makes it part of this type's definition: every+ # instance gets it for free, with no separate import-time attachment+ # step (the bare-function equivalent, ``fn._fail_on_errors = ...``+ # after ``def fn(url): ...``) to forget on a future rewrite. That is+ # not the same as being un-droppable — a rebind through a lambda,+ # ``functools.partial``, or an un-attribute-aware wrapper would still+ # lose it, and ``functools.wraps`` -- which copies a *function's*+ # ``__dict__`` and so kept the old bare-function attribute -- does+ # not copy this class attribute. The actual guard is+ # ``test_fetchers_expose_the_fail_on_errors_contract`` in+ # ``tests/web/services/test_pdf_service.py``, which asserts+ # ``getattr(fetcher, "_fail_on_errors", None) is False`` on the wired+ # module-level ``_safe_url_fetcher``/``_skipping_url_fetcher``+ # objects. ``False`` mirrors the+ # ``URLFetcher(fail_on_errors=False)`` default: the refusal surfaces+ # as ``URLFetchingError``, which WeasyPrint's callers warn-and-skip.+ # That keeps the 68.x ValueError-as-skip posture (pinned end-to-end+ # by ``test_render_succeeds_when_body_url_is_blocked``) while the+ # version floor is ~=70.0 for the GHSA-jf6q-chmf-3h3v+ # url_fetcher-bypass fix.+ _fail_on_errors = False++ def __call__(self, url):+ if not validate_url(url):+ # A rejected URL is adversarial-shaped and may carry the+ # operator's real credentials (RFC 3986 userinfo) or secret+ # query tokens; log and raise only scheme://host:port, the+ # same discipline every ssrf_validator site follows.+ redacted = redact_url_for_log(url)+ logger.warning(f"Blocked unsafe URL in PDF rendering: {redacted}")+ raise UnsafePDFResourceURLError(+ f"Blocked unsafe URL in PDF rendering: {redacted}"+ )+ _ensure_weasyprint()+ return _URL_FETCHER.fetch(url)+++_safe_url_fetcher = _SafeUrlFetcher()-def _skipping_url_fetcher(url):++class _SkippingUrlFetcher:"""As `_safe_url_fetcher`, but a refusal yields an empty resource.Used only by the retry in `_render_pdf`. No fetch happens for a URL the@@ -142,39 +173,41 @@ def _skipping_url_fetcher(url):what WeasyPrint is told afterwards: nothing, rather than an exception ithandles inconsistently."""- try:- return _safe_url_fetcher(url)- except Exception:- # Same redaction discipline as the fetcher's own block path:- # a refused/unavailable URL is adversarial-shaped and may carry- # credentials in userinfo or secret query tokens.- logger.warning(- f"Skipping unavailable resource in PDF rendering: {redact_url_for_log(url)}"- )- if URLFetcherResponse is not None:- return URLFetcherResponse(- url,- body=b"",- headers={"Content-Type": _EMPTY_RESOURCE_MIME_TYPE},++ # Unlike `_SafeUrlFetcher`, this fetcher's `__call__` never lets an+ # exception escape -- it always returns a resource -- so WeasyPrint's+ # `fetch()` never hits the except-branch that reads `_fail_on_errors`+ # off this instance; it is dead for this class at runtime. Kept as a+ # class attribute anyway so both fetchers satisfy the same contract+ # test (`test_fetchers_expose_the_fail_on_errors_contract`) and so a+ # future edit that makes `__call__` raise again inherits a safe+ # default instead of an `AttributeError`.+ _fail_on_errors = False++ def __call__(self, url):+ try:+ return _safe_url_fetcher(url)+ except Exception:+ # Same redaction discipline as the fetcher's own block path:+ # a refused/unavailable URL is adversarial-shaped and may carry+ # credentials in userinfo or secret query tokens.+ logger.warning(+ f"Skipping unavailable resource in PDF rendering: {redact_url_for_log(url)}")- return {- "string": b"",- "mime_type": _EMPTY_RESOURCE_MIME_TYPE,- "redirected_url": url,- }---# WeasyPrint 70's fetch() context manager (weasyprint/urls.py) reads-# ``_fail_on_errors`` off the *url_fetcher itself* after catching an exception-# it raised — a bare function without that attribute turns every blocked URL-# into an ``AttributeError`` mid-render. ``False`` mirrors the-# ``URLFetcher(fail_on_errors=False)`` default: the refusal surfaces as-# ``URLFetchingError``, which WeasyPrint's callers warn-and-skip. That keeps-# the 68.x ValueError-as-skip posture (pinned end-to-end by-# ``test_render_succeeds_when_body_url_is_blocked``) while the version floor-# is ~=70.0 for the GHSA-jf6q-chmf-3h3v url_fetcher-bypass fix.-_safe_url_fetcher._fail_on_errors = False-_skipping_url_fetcher._fail_on_errors = False+ if URLFetcherResponse is not None:+ return URLFetcherResponse(+ url,+ body=b"",+ headers={"Content-Type": _EMPTY_RESOURCE_MIME_TYPE},+ )+ return {+ "string": b"",+ "mime_type": _EMPTY_RESOURCE_MIME_TYPE,+ "redirected_url": url,+ }+++_skipping_url_fetcher = _SkippingUrlFetcher()class MissingPDFDependencyError(RuntimeError):diff --git a/tests/security/test_ssrf_egress_policy.py b/tests/security/test_ssrf_egress_policy.pyindex f306a7576f..166de8f199 100644--- a/tests/security/test_ssrf_egress_policy.py+++ b/tests/security/test_ssrf_egress_policy.py@@ -258,18 +258,45 @@ def _called_names(source: str, func_name: str) -> set[str]:Nesting matters: ``api_add_resource`` does its work in a nested``_impl()`` that it hands to a worker thread, so a body-only scan- would miss the gate.+ would miss the gate. ``func_name`` may also name a class (a+ callable-class WeasyPrint fetcher), scanning every method the same+ way -- or, as ``"ClassName.method_name"``, a single method inside a+ class, when a census entry needs to pin the gate to the one method+ that actually runs it rather than to the class as a whole."""tree = ast.parse(source)- target = None- for node in ast.walk(tree):- if (- isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))- and node.name == func_name- ):- target = node- break- assert target is not None, f"function {func_name!r} not found"+ if "." in func_name:+ class_name, method_name = func_name.split(".", 1)+ class_node = None+ for node in ast.walk(tree):+ if isinstance(node, ast.ClassDef) and node.name == class_name:+ class_node = node+ break+ assert class_node is not None, f"class {class_name!r} not found"++ target = None+ for node in ast.walk(class_node):+ if (+ isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))+ and node.name == method_name+ ):+ target = node+ break+ assert target is not None, (+ f"method {method_name!r} not found in class {class_name!r}"+ )+ else:+ target = None+ for node in ast.walk(tree):+ if (+ isinstance(+ node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)+ )+ and node.name == func_name+ ):+ target = node+ break+ assert target is not None, f"function {func_name!r} not found"names: set[str] = set()for node in ast.walk(target):@@ -300,6 +327,23 @@ _NESTED_GATE_PROBE = textwrap.dedent(""")+# A class with two methods, only one of which gates -- the shape+# ``_SafeUrlFetcher``/``_SkippingUrlFetcher`` share in pdf_service.py, and+# what the dotted "ClassName.method_name" form of ``_called_names`` needs+# to tell apart.+_CLASS_MIXED_METHODS_PROBE = textwrap.dedent(+ """+ class Handler:+ def gated(self, url):+ if not validate_url(url):+ return None+ return fetch_it(url)++ def ungated(self, url):+ return fetch_it(url)+ """+)+def test_gate_locator_distinguishes_gated_from_ungated():"""Control for the parametrised census below."""@@ -307,6 +351,26 @@ def test_gate_locator_distinguishes_gated_from_ungated():assert "validate_url" in _called_names(_NESTED_GATE_PROBE, "handler")+def test_gate_locator_scopes_dotted_name_to_one_method():+ """Control for the dotted ``ClassName.method_name`` form.++ Scanning the whole class (undotted) would find ``validate_url`` via+ ``gated`` even when asked about ``ungated`` -- exactly the false+ pass a census entry pinned to a specific method (like+ ``_SafeUrlFetcher.__call__``) needs to avoid.+ """+ assert "validate_url" in _called_names(+ _CLASS_MIXED_METHODS_PROBE, "Handler.gated"+ )+ assert "validate_url" not in _called_names(+ _CLASS_MIXED_METHODS_PROBE, "Handler.ungated"+ )+ # Sanity: the undotted, whole-class form still sees both.+ assert "validate_url" in _called_names(+ _CLASS_MIXED_METHODS_PROBE, "Handler"+ )++# (module, handler, gate) for every place a user-supplied URL — request# body, or a settings value the user edits through the UI — reaches an# outbound request directly in the ported web layer. Available-model discovery@@ -345,8 +409,18 @@ _GATED_ENTRY_POINTS = ["_reject_custom_endpoint","is_safe_custom_llm_endpoint",),- # WeasyPrint resource fetching during PDF export.- ("web/services/pdf_service.py", "_safe_url_fetcher", "validate_url"),+ # WeasyPrint resource fetching during PDF export. Pinned to+ # ``__call__`` specifically, not the whole class: ``_SafeUrlFetcher``+ # also carries a ``_fail_on_errors`` class attribute (see #6685 for+ # why it's a class rather than a bare function), and a whole-class+ # scan would keep passing even if a refactor moved the+ # ``validate_url`` call out of ``__call__`` while leaving some other+ # attribute or method referencing the name.+ (+ "web/services/pdf_service.py",+ "_SafeUrlFetcher.__call__",+ "validate_url",+ ),]diff --git a/tests/web/services/test_pdf_service.py b/tests/web/services/test_pdf_service.pyindex 6f32f09b4c..e4363c9a43 100644--- a/tests/web/services/test_pdf_service.py+++ b/tests/web/services/test_pdf_service.py@@ -579,6 +579,68 @@ class TestSafeUrlFetcher:is pdf_module._safe_url_fetcher)+ def test_fetchers_expose_the_fail_on_errors_contract(self):+ """Both fetchers must expose ``_fail_on_errors = False``.++ WeasyPrint 70's ``fetch()`` reads this private attribute off the+ ``url_fetcher`` itself after catching an exception it raised; a+ callable that lost it turns every blocked URL into an+ ``AttributeError`` mid-render. The contract rides on the fetcher+ classes as a class attribute so every instance carries it as part+ of the type -- there is no separate import-time attachment step+ (the bare-function equivalent, ``fn._fail_on_errors = ...``) to+ forget on a rewrite. It can still be lost through a rebind (a+ lambda, ``functools.partial``, or an un-attribute-aware wrapper+ all drop it, and ``functools.wraps`` -- which *does* preserve a+ bare function's own attribute -- does not copy an instance's).+ This test is the actual guard: it asserts the attribute is+ present, ``False``, on the wired module-level fetcher objects+ (#6685).+ """+ from local_deep_research.web.services.pdf_service import (+ _SafeUrlFetcher,+ _SkippingUrlFetcher,+ _safe_url_fetcher,+ _skipping_url_fetcher,+ )++ for fetcher in (_safe_url_fetcher, _skipping_url_fetcher):+ assert getattr(fetcher, "_fail_on_errors", None) is False+ # Intrinsic to the type, not attached to the callable afterwards.+ assert _SafeUrlFetcher._fail_on_errors is False+ assert _SkippingUrlFetcher._fail_on_errors is False++ def test_installed_weasyprint_still_reads_fail_on_errors(self):+ """Tripwire for the upstream half of the contract (#6685).++ The class attribute exists because WeasyPrint 70's ``fetch()``+ reads ``_fail_on_errors`` off the url_fetcher after catching an+ exception it raised. If a future WeasyPrint renames or drops+ that private lookup, the attribute becomes dead configuration+ and refusal semantics change silently at runtime — this pin+ fails loudly in CI so the contract gets re-derived first.++ Scoped to ``fetch``'s own source, not the whole ``urls`` module:+ ``URLFetcher.__init__`` also assigns ``self._fail_on_errors = ...``+ (it's the constructor argument backing it), so a module-wide+ source scan for the string would still pass even if the lookup+ this test actually cares about -- the one inside ``fetch()`` that+ turns a blocked URL into ``FatalURLFetchingError`` vs.+ ``URLFetchingError`` -- were renamed or dropped.+ """+ import inspect++ weasyprint_urls = pytest.importorskip("weasyprint.urls")++ fetch = getattr(weasyprint_urls, "fetch", None)+ assert inspect.isfunction(fetch), (+ "weasyprint.urls has no module-level fetch() function; "+ "the _fail_on_errors contract this pins may have moved -- "+ "re-derive it against the installed WeasyPrint version."+ )++ assert "_fail_on_errors" in inspect.getsource(fetch)+def test_render_succeeds_when_body_url_is_blocked(self):"""End-to-end: a malicious URL in the markdown body doesn't break rendering.