~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Caller-safe download skip-reason tokens (single source of truth).
PR #6564 follow-up: ``research_library.services.download_service`` and
``research_library.downloaders.html`` each carried their own copy of the
exception-class -> fixed-token table. The copies drifted (``DataError``,
``InterfaceError``, ``InvalidRequestError`` and the ``OSError`` errno branch
existed only on the services side), and the "mirrors" comment overstated the
parity. This module owns the table and the mapping function; both call sites
import from here so they cannot drift again.
Contract (CWE-209): never pass ``str(exc)`` to the caller. The exception text
can carry SQL text (``INSERT INTO document_collections ...``), target URLs,
or absolute file paths. Caller-visible ``skip_reason`` values are fixed
tokens or ``download_error:<ClassName>`` only. Server-side logs retain the
exception class; ``DownloadAttempt.error_message`` stores scrubbed detail.
``sanitize_error_for_client()`` is NOT a substitute — it scrubs credential
shapes only, not SQL/paths/URLs.
"""
import errno
import os
# Client-safe skip-reason mapping for download exception paths.
#
# sanitize_error_for_client() scrubs credential *shapes* but explicitly does
# NOT remove SQL text, provider endpoints, or dependency internals. The
# exception's text here can be a SQLAlchemy IntegrityError
# ("...INSERT INTO document_collections..."), a requests ConnectionError
# carrying the target host, or a PDF library message with absolute paths.
CLIENT_SAFE_DOWNLOAD_MESSAGES: dict[str, str] = {
# Network surface — never echo the URL / host the caller already knows.
"ConnectionError": "network_unavailable",
"ConnectionResetError": "network_reset",
"ConnectionAbortedError": "network_reset",
"BrokenPipeError": "network_reset",
"ConnectionRefusedError": "network_unavailable",
"Timeout": "network_timeout",
# Not reached via client_safe_download_message (``_network_token`` returns
# first for every TimeoutError); kept so direct consumers of this map, such
# as the classifier vocabulary test, see the token.
"TimeoutError": "network_timeout",
"HTTPError": "http_error",
"ChunkedEncodingError": "network_unavailable",
"ContentDecodingError": "network_unavailable",
"TooManyRedirects": "network_redirect_limit",
# SQL surface — never echo SQL text, table names, or driver internals.
# Maps to the SQLAlchemy class names; the IntegrityError case is the
# one the FK-ON regression surfaced ("FOREIGN KEY constraint failed
# [SQL: INSERT INTO document_collections ...]").
"IntegrityError": "database_constraint",
"OperationalError": "database_unavailable",
"DatabaseError": "database_error",
"DataError": "database_data_error",
"InterfaceError": "database_interface_error",
"InvalidRequestError": "database_invalid_request",
# PDF / extraction surface — never echo absolute paths or parser internals.
"PDFStorageLimitError": "pdf_storage_size_limit",
"PdfReadError": "pdf_parse_failed",
"PdfStreamError": "pdf_parse_failed",
}
# Network-layer failures that are not ``ConnectionError`` subclasses but would
# otherwise be swallowed by the OSError errno branch below (and surface as
# ``filesystem_error:<strerror>`` with the 1h filesystem cooldown) or fall to
# ``download_error:<ClassName>``. Matched subclass-aware via the MRO so
# ``requests.exceptions.SSLError``, ``ssl.SSLEOFError`` and friends all hit.
# ``ConnectionError`` / ``TimeoutError`` themselves are handled separately in
# ``_network_token``.
_NETWORK_CLASS_TOKENS: dict[str, str] = {
"SSLError": "network_unavailable", # ssl.SSLError, requests SSLError, ...
"gaierror": "network_unavailable", # DNS: socket.gaierror
"herror": "network_unavailable", # DNS: socket.herror
"RemoteDisconnected": "network_reset", # server closed mid-download
"IncompleteRead": "network_reset",
"ReadTimeoutError": "network_timeout", # urllib3
"ConnectTimeoutError": "network_timeout",
"ReadTimeout": "network_timeout", # requests (what the download path raises)
"ConnectTimeout": "network_timeout",
}
# Canonical network-layer errnos on plain ``OSError`` (no dedicated subclass).
# Built with getattr so platform-missing names (Windows/WSA*) are skipped.
_NETWORK_UNAVAILABLE_ERRNOS: frozenset[int] = frozenset(
getattr(errno, name)
for name in (
"ENETUNREACH",
"ENETDOWN",
"EHOSTUNREACH",
"EHOSTDOWN",
"ENETRESET",
"ENOTCONN",
"EADDRNOTAVAIL",
"ECONNREFUSED",
# Windows (WSA*) spellings; skipped on platforms without them.
"WSAECONNREFUSED",
"WSAENETUNREACH",
"WSAENETDOWN",
"WSAEHOSTUNREACH",
"WSAEHOSTDOWN",
"WSAENETRESET",
"WSAENOTCONN",
"WSAEADDRNOTAVAIL",
)
if hasattr(errno, name)
)
_NETWORK_RESET_ERRNOS: frozenset[int] = frozenset(
getattr(errno, name)
for name in (
"ECONNRESET",
"ECONNABORTED",
"EPIPE", # matches BrokenPipeError -> network_reset
"WSAECONNRESET",
"WSAECONNABORTED",
)
if hasattr(errno, name)
)
_NETWORK_TIMEOUT_ERRNOS: frozenset[int] = frozenset(
getattr(errno, name)
for name in ("ETIMEDOUT", "WSAETIMEDOUT")
if hasattr(errno, name)
)
def _network_token(exc: BaseException) -> str | None:
"""Return a fixed network token when *exc* is a network-layer failure.
Checked before the OSError errno branch so the network errnos listed in
the ``_NETWORK_*_ERRNOS`` sets (including their Windows ``WSA*`` spellings
where the platform defines them) never reach ``filesystem_error:`` (1h
cooldown). Returns ``None`` for everything else.
"""
for cls in type(exc).__mro__:
token = _NETWORK_CLASS_TOKENS.get(cls.__name__)
if token:
return token
if isinstance(exc, TimeoutError):
return "network_timeout"
if isinstance(exc, ConnectionResetError):
return "network_reset"
if isinstance(exc, ConnectionError):
# ConnectionRefused/Aborted/BrokenPipe etc. (never filesystem)
return CLIENT_SAFE_DOWNLOAD_MESSAGES.get(
type(exc).__name__, "network_unavailable"
)
if isinstance(exc, OSError) and exc.errno is not None:
if exc.errno in _NETWORK_TIMEOUT_ERRNOS:
return "network_timeout"
if exc.errno in _NETWORK_RESET_ERRNOS:
return "network_reset"
if exc.errno in _NETWORK_UNAVAILABLE_ERRNOS:
return "network_unavailable"
return None
def client_safe_download_message(exc: BaseException) -> str:
"""Return a caller-safe skip_reason for *exc*.
Author-written constant for known classes; the exception's class name
for everything else. Never passes ``str(exc)`` through — that can carry
SQL text, target URLs, or file paths the caller should not see. The
downloader service logs only the exception class and stores a
credential-scrubbed message in ``DownloadAttempt.error_message`` (see its
outer PDF handler).
``OSError`` is the one exception family where ``str(exc)`` is *not*
free of app-controlled text — Python formats it as
``"[Errno N] <strerror>: <filename>"`` and the filename field is
arbitrary. We can't safely pass ``str(exc)`` through. What we *can*
safely surface is ``exc.strerror`` when it matches the OS-level
canonical message (i.e. ``os.strerror(exc.errno)``). That string is
provided by the OS — not by the application — so it carries no SQL,
URL, or path content. Custom constructions like
``OSError(28, "leak: secret file contents")`` deliberately don't match the
canonical message and fall through to the class-name default.
Network-layer failures never reach the errno branch: ``_network_token``
runs first and maps the ``ConnectionError`` family, ``TimeoutError``
(including ``ETIMEDOUT``), the ENET*/EHOST* errnos, SSL, DNS and
``RemoteDisconnected`` failures to fixed ``network_*`` tokens. Only errnos
outside the network families that ``_network_token`` recognizes (for
example ``ENOSPC`` or ``ENOENT``) take the ``filesystem_error:`` branch.
"""
network = _network_token(exc)
if network is not None:
return network
if isinstance(exc, OSError) and exc.errno is not None:
try:
canonical = os.strerror(exc.errno)
except (ValueError, OverflowError):
canonical = None
if canonical and exc.strerror and exc.strerror == canonical:
return f"filesystem_error:{exc.strerror}"
return CLIENT_SAFE_DOWNLOAD_MESSAGES.get(
type(exc).__name__, f"download_error:{type(exc).__name__}"
)
# What a person reads for a skip-reason token they can act on themselves.
# The tokens stay what the services return and the retry manager classifies;
# the routes that show a download result to a person (the bulk download
# stream and the single-download endpoints) show this text instead.
CLIENT_SAFE_DOWNLOAD_USER_MESSAGES: dict[str, str] = {
"pdf_storage_size_limit": (
"The PDF is larger than the PDF storage size limit, so it was not "
"stored (its text is kept when it could be extracted). Raise "
'"Maximum PDF Size (MB)" in the research library settings to '
"store it."
),
}
def download_reason_for_user(reason):
"""Return the text to show a person for a download skip *reason*.
A token listed in ``CLIENT_SAFE_DOWNLOAD_USER_MESSAGES`` becomes its
message; any other value is returned unchanged.
"""
if isinstance(reason, str):
return CLIENT_SAFE_DOWNLOAD_USER_MESSAGES.get(reason, reason)
return reason