Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

9.5 KB
Raw Blame
"""Caller-safe download skip-reason tokens (single source of truth).

PR #6564 follow-up: ``research_library.services.download_service`` and
``research_library.downloaders.html`` each carried their own copy of the
exception-class -> fixed-token table. The copies drifted (``DataError``,
``InterfaceError``, ``InvalidRequestError`` and the ``OSError`` errno branch
existed only on the services side), and the "mirrors" comment overstated the
parity. This module owns the table and the mapping function; both call sites
import from here so they cannot drift again.

Contract (CWE-209): never pass ``str(exc)`` to the caller. The exception text
can carry SQL text (``INSERT INTO document_collections ...``), target URLs,
or absolute file paths. Caller-visible ``skip_reason`` values are fixed
tokens or ``download_error:<ClassName>`` only. Server-side logs retain the
exception class; ``DownloadAttempt.error_message`` stores scrubbed detail.
``sanitize_error_for_client()`` is NOT a substitute — it scrubs credential
shapes only, not SQL/paths/URLs.
"""

import errno
import os

# Client-safe skip-reason mapping for download exception paths.
#
# sanitize_error_for_client() scrubs credential *shapes* but explicitly does
# NOT remove SQL text, provider endpoints, or dependency internals. The
# exception's text here can be a SQLAlchemy IntegrityError
# ("...INSERT INTO document_collections..."), a requests ConnectionError
# carrying the target host, or a PDF library message with absolute paths.
CLIENT_SAFE_DOWNLOAD_MESSAGES: dict[str, str] = {
    # Network surface — never echo the URL / host the caller already knows.
    "ConnectionError": "network_unavailable",
    "ConnectionResetError": "network_reset",
    "ConnectionAbortedError": "network_reset",
    "BrokenPipeError": "network_reset",
    "ConnectionRefusedError": "network_unavailable",
    "Timeout": "network_timeout",
    # Not reached via client_safe_download_message (``_network_token`` returns
    # first for every TimeoutError); kept so direct consumers of this map, such
    # as the classifier vocabulary test, see the token.
    "TimeoutError": "network_timeout",
    "HTTPError": "http_error",
    "ChunkedEncodingError": "network_unavailable",
    "ContentDecodingError": "network_unavailable",
    "TooManyRedirects": "network_redirect_limit",
    # SQL surface — never echo SQL text, table names, or driver internals.
    # Maps to the SQLAlchemy class names; the IntegrityError case is the
    # one the FK-ON regression surfaced ("FOREIGN KEY constraint failed
    # [SQL: INSERT INTO document_collections ...]").
    "IntegrityError": "database_constraint",
    "OperationalError": "database_unavailable",
    "DatabaseError": "database_error",
    "DataError": "database_data_error",
    "InterfaceError": "database_interface_error",
    "InvalidRequestError": "database_invalid_request",
    # PDF / extraction surface — never echo absolute paths or parser internals.
    "PDFStorageLimitError": "pdf_storage_size_limit",
    "PdfReadError": "pdf_parse_failed",
    "PdfStreamError": "pdf_parse_failed",
}

# Network-layer failures that are not ``ConnectionError`` subclasses but would
# otherwise be swallowed by the OSError errno branch below (and surface as
# ``filesystem_error:<strerror>`` with the 1h filesystem cooldown) or fall to
# ``download_error:<ClassName>``. Matched subclass-aware via the MRO so
# ``requests.exceptions.SSLError``, ``ssl.SSLEOFError`` and friends all hit.
# ``ConnectionError`` / ``TimeoutError`` themselves are handled separately in
# ``_network_token``.
_NETWORK_CLASS_TOKENS: dict[str, str] = {
    "SSLError": "network_unavailable",  # ssl.SSLError, requests SSLError, ...
    "gaierror": "network_unavailable",  # DNS: socket.gaierror
    "herror": "network_unavailable",  # DNS: socket.herror
    "RemoteDisconnected": "network_reset",  # server closed mid-download
    "IncompleteRead": "network_reset",
    "ReadTimeoutError": "network_timeout",  # urllib3
    "ConnectTimeoutError": "network_timeout",
    "ReadTimeout": "network_timeout",  # requests (what the download path raises)
    "ConnectTimeout": "network_timeout",
}

# Canonical network-layer errnos on plain ``OSError`` (no dedicated subclass).
# Built with getattr so platform-missing names (Windows/WSA*) are skipped.
_NETWORK_UNAVAILABLE_ERRNOS: frozenset[int] = frozenset(
    getattr(errno, name)
    for name in (
        "ENETUNREACH",
        "ENETDOWN",
        "EHOSTUNREACH",
        "EHOSTDOWN",
        "ENETRESET",
        "ENOTCONN",
        "EADDRNOTAVAIL",
        "ECONNREFUSED",
        # Windows (WSA*) spellings; skipped on platforms without them.
        "WSAECONNREFUSED",
        "WSAENETUNREACH",
        "WSAENETDOWN",
        "WSAEHOSTUNREACH",
        "WSAEHOSTDOWN",
        "WSAENETRESET",
        "WSAENOTCONN",
        "WSAEADDRNOTAVAIL",
    )
    if hasattr(errno, name)
)
_NETWORK_RESET_ERRNOS: frozenset[int] = frozenset(
    getattr(errno, name)
    for name in (
        "ECONNRESET",
        "ECONNABORTED",
        "EPIPE",  # matches BrokenPipeError -> network_reset
        "WSAECONNRESET",
        "WSAECONNABORTED",
    )
    if hasattr(errno, name)
)
_NETWORK_TIMEOUT_ERRNOS: frozenset[int] = frozenset(
    getattr(errno, name)
    for name in ("ETIMEDOUT", "WSAETIMEDOUT")
    if hasattr(errno, name)
)


def _network_token(exc: BaseException) -> str | None:
    """Return a fixed network token when *exc* is a network-layer failure.

    Checked before the OSError errno branch so the network errnos listed in
    the ``_NETWORK_*_ERRNOS`` sets (including their Windows ``WSA*`` spellings
    where the platform defines them) never reach ``filesystem_error:`` (1h
    cooldown). Returns ``None`` for everything else.
    """
    for cls in type(exc).__mro__:
        token = _NETWORK_CLASS_TOKENS.get(cls.__name__)
        if token:
            return token
    if isinstance(exc, TimeoutError):
        return "network_timeout"
    if isinstance(exc, ConnectionResetError):
        return "network_reset"
    if isinstance(exc, ConnectionError):
        # ConnectionRefused/Aborted/BrokenPipe etc. (never filesystem)
        return CLIENT_SAFE_DOWNLOAD_MESSAGES.get(
            type(exc).__name__, "network_unavailable"
        )
    if isinstance(exc, OSError) and exc.errno is not None:
        if exc.errno in _NETWORK_TIMEOUT_ERRNOS:
            return "network_timeout"
        if exc.errno in _NETWORK_RESET_ERRNOS:
            return "network_reset"
        if exc.errno in _NETWORK_UNAVAILABLE_ERRNOS:
            return "network_unavailable"
    return None


def client_safe_download_message(exc: BaseException) -> str:
    """Return a caller-safe skip_reason for *exc*.

    Author-written constant for known classes; the exception's class name
    for everything else. Never passes ``str(exc)`` through — that can carry
    SQL text, target URLs, or file paths the caller should not see. The
    downloader service logs only the exception class and stores a
    credential-scrubbed message in ``DownloadAttempt.error_message`` (see its
    outer PDF handler).

    ``OSError`` is the one exception family where ``str(exc)`` is *not*
    free of app-controlled text — Python formats it as
    ``"[Errno N] <strerror>: <filename>"`` and the filename field is
    arbitrary. We can't safely pass ``str(exc)`` through. What we *can*
    safely surface is ``exc.strerror`` when it matches the OS-level
    canonical message (i.e. ``os.strerror(exc.errno)``). That string is
    provided by the OS — not by the application — so it carries no SQL,
    URL, or path content. Custom constructions like
    ``OSError(28, "leak: secret file contents")`` deliberately don't match the
    canonical message and fall through to the class-name default.

    Network-layer failures never reach the errno branch: ``_network_token``
    runs first and maps the ``ConnectionError`` family, ``TimeoutError``
    (including ``ETIMEDOUT``), the ENET*/EHOST* errnos, SSL, DNS and
    ``RemoteDisconnected`` failures to fixed ``network_*`` tokens. Only errnos
    outside the network families that ``_network_token`` recognizes (for
    example ``ENOSPC`` or ``ENOENT``) take the ``filesystem_error:`` branch.
    """
    network = _network_token(exc)
    if network is not None:
        return network
    if isinstance(exc, OSError) and exc.errno is not None:
        try:
            canonical = os.strerror(exc.errno)
        except (ValueError, OverflowError):
            canonical = None
        if canonical and exc.strerror and exc.strerror == canonical:
            return f"filesystem_error:{exc.strerror}"
    return CLIENT_SAFE_DOWNLOAD_MESSAGES.get(
        type(exc).__name__, f"download_error:{type(exc).__name__}"
    )


# What a person reads for a skip-reason token they can act on themselves.
# The tokens stay what the services return and the retry manager classifies;
# the routes that show a download result to a person (the bulk download
# stream and the single-download endpoints) show this text instead.
CLIENT_SAFE_DOWNLOAD_USER_MESSAGES: dict[str, str] = {
    "pdf_storage_size_limit": (
        "The PDF is larger than the PDF storage size limit, so it was not "
        "stored (its text is kept when it could be extracted). Raise "
        '"Maximum PDF Size (MB)" in the research library settings to '
        "store it."
    ),
}


def download_reason_for_user(reason):
    """Return the text to show a person for a download skip *reason*.

    A token listed in ``CLIENT_SAFE_DOWNLOAD_USER_MESSAGES`` becomes its
    message; any other value is returned unchanged.
    """
    if isinstance(reason, str):
        return CLIENT_SAFE_DOWNLOAD_USER_MESSAGES.get(reason, reason)
    return reason