Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

24.8 KB
Raw Blame
"""
Centralized path validation utilities for security.

This module provides secure path validation to prevent path traversal attacks
and other filesystem-based security vulnerabilities.
"""

import errno
import ntpath
import os
import posixpath
import re
import unicodedata
from pathlib import Path
from typing import Optional, Union
from urllib.parse import unquote

from loguru import logger
from werkzeug.security import safe_join

from ..config.paths import get_models_directory
from .log_sanitizer import redact_and_bound_for_log


# Encoded forms of ".." and "." that path traversal attacks use to bypass
# naive `..`-substring checks. The list covers both single and double URL
# encoding (where the % itself is encoded as %25).
_ENCODED_TRAVERSAL_TOKENS = (
    "%2e%2e",  # ..
    "%2E%2E",
    "%2e.",  # .. (mixed)
    ".%2e",
    "%252e%252e",  # double-encoded ..
    "%252E%252E",
    "%2f",  # /
    "%2F",
)


def _has_encoded_traversal(text: str) -> bool:
    """Return True if `text` contains URL-encoded path-traversal tokens.

    Checks for both single and double URL encoding of '..', '.', and '/'.
    Decoders downstream may convert these into real '..' segments that
    escape the base directory; reject the input before that happens.
    """
    lowered = text.lower()
    for tok in _ENCODED_TRAVERSAL_TOKENS:
        if tok.lower() in lowered:
            return True
    # Recursive decode: if unquote changes the string, the original
    # contained encoded characters. Re-check the decoded form for ".."
    # in case the encoding used is one not enumerated above.
    decoded = unquote(text)
    if decoded != text and ".." in decoded:
        return True
    # Double-decode catches %252e%252e -> %2e%2e -> ..
    double_decoded = unquote(decoded)
    if double_decoded != decoded and ".." in double_decoded:
        return True
    return False


def _has_unicode_traversal(text: str) -> bool:
    """Return True if `text` contains unicode look-alikes for path traversal.

    Many unicode characters NFKC-normalize to '.' or '/', so an attacker
    can use full-width periods (U+FF0E '.') or other look-alikes to
    bypass naive '..'-substring checks. After normalization, '..' or
    '/..' segments indicate an attempt to traverse.
    """
    normalized = unicodedata.normalize("NFKC", text)
    if normalized == text:
        return False
    return (
        ".." in normalized
        or normalized.startswith("/..")
        or "/.." in normalized
    )


# ``os.path.ALLOW_MISSING`` exists on Python 3.14 and was backported to
# 3.12.11 and 3.13.4. Older patch releases use the fallback below. It is
# read from the platform's path module, picked the way ``os`` picks
# ``os.path``, because the check-pathlib-usage hook forbids ``os.path``
# and the sentinel has no pathlib spelling.
_PLATFORM_PATH_MODULE = ntpath if os.name == "nt" else posixpath
_ALLOW_MISSING = getattr(_PLATFORM_PATH_MODULE, "ALLOW_MISSING", None)


def resolve_allow_missing(path: Union[str, Path]) -> Path:
    """Resolve every symlink in ``path``. Only a missing tail is tolerated.

    Use this, never a bare ``Path.resolve()``, wherever a resolved path is
    compared against a containment boundary. Non-strict ``resolve()``
    ignores every ``OSError`` that ``lstat`` raises, not only "does not
    exist". A short in-base symlink to a deep in-base directory chain makes
    the resolved prefix longer than PATH_MAX, so every later ``lstat`` fails
    with ENAMETOOLONG. The remaining components, including a symlink that
    leaves the base, are then appended without being resolved. The result
    looks contained, but the kernel resolves the short path without that
    limit and follows the link out (the CVE-2025-4517 class).

    Here a component that does not exist is appended as written, so a leaf
    that has not been created yet still resolves. Any other failure raises
    ``OSError``: ENAMETOOLONG, ELOOP (a symlink loop), EACCES, and ENOTDIR
    (a path that continues below a regular file). Python 3.12 reports a
    loop as ``RuntimeError``, which is re-raised as ``OSError(ELOOP)``.

    Without ``os.path.ALLOW_MISSING`` the longest prefix whose last entry
    exists is resolved strictly and the missing tail is appended. A dangling
    symlink, or a ``..`` in the missing tail, raises there as well. Both
    are refusals that the ``ALLOW_MISSING`` path would not make, so the
    fallback is only ever stricter.
    """
    if isinstance(path, str):
        path = Path(path)
    try:
        if _ALLOW_MISSING is not None:
            return path.resolve(strict=_ALLOW_MISSING)
        return _resolve_allow_missing_fallback(path)
    except RuntimeError as exc:
        raise OSError(errno.ELOOP, "Symlink loop", str(path)) from exc


def _resolve_allow_missing_fallback(path: Path) -> Path:
    """``resolve_allow_missing`` for a Python without ``ALLOW_MISSING``."""
    path = path.absolute()
    missing: list[str] = []
    head = path
    while True:
        try:
            # lstat, not exists(): only ENOENT means "missing". Every
            # other error (ENAMETOOLONG, ENOTDIR, EACCES) propagates.
            os.lstat(head)
            break
        except FileNotFoundError:
            if head.parent == head:
                raise
            missing.append(head.name)
            head = head.parent
    if ".." in missing:
        # ".." below a missing component cannot be resolved against the
        # filesystem, and is_relative_to() would compare it lexically.
        raise OSError(errno.ENOENT, "Unresolvable '..' segment", str(path))
    # strict=True raises for a dangling symlink, a loop and ENAMETOOLONG.
    return head.resolve(strict=True).joinpath(*reversed(missing))


class PathValidator:
    """Centralized path validation for security."""

    # Regex for safe filename/path characters
    SAFE_PATH_PATTERN = re.compile(r"^[a-zA-Z0-9._/\-]+$")

    # Allowed config file extensions
    CONFIG_EXTENSIONS = (".json", ".yaml", ".yml", ".toml", ".ini", ".conf")

    @staticmethod
    def validate_safe_path(
        user_input: str,
        base_dir: Union[str, Path],
        allow_absolute: bool = False,
        required_extensions: Optional[tuple] = None,
    ) -> Optional[Path]:
        """
        Validate and sanitize a user-provided path.

        Args:
            user_input: The user-provided path string
            base_dir: The safe base directory to contain paths within
            allow_absolute: Whether to allow absolute paths (with restrictions)
            required_extensions: Tuple of required file extensions (e.g., ('.json', '.yaml'))

        Returns:
            The validated Path. It is never None: every rejection raises.

        Raises:
            ValueError: If the path is invalid or unsafe
        """
        if not user_input or not isinstance(user_input, str):
            raise ValueError("Invalid path input")

        if "\x00" in user_input:
            raise ValueError("Null bytes are not allowed in path")

        # Strip whitespace
        user_input = user_input.strip()
        # (#6938: the input is a client-chosen path — e.g. the
        # unauthenticated /static route — so the rejection log lines
        # below interpolate a redacted, bounded form, never the raw value.
        # It is built only in those rejection branches: this runs on every
        # static-file lookup, and redaction costs up to tens of
        # milliseconds on a long adversarial path.)

        # Reject URL-encoded traversal tokens (single or double encoded).
        # safe_join's literal-".."-check doesn't catch %2e%2e or %252e%252e,
        # so do it explicitly here before any decoding takes place.
        if _has_encoded_traversal(user_input):
            logger.warning(
                f"Encoded path-traversal attempt blocked: {redact_and_bound_for_log(user_input, 200)!r}"
            )
            raise ValueError(
                "Invalid path - encoded traversal pattern detected"
            )

        # Reject unicode look-alike traversal (e.g. full-width '..').
        # NFKC normalization is the same form most filesystems / browsers
        # apply; if normalizing produces a '..' segment the original was a
        # disguised traversal attempt.
        if _has_unicode_traversal(user_input):
            logger.warning(
                f"Unicode path-traversal attempt blocked: {redact_and_bound_for_log(user_input, 200)!r}"
            )
            raise ValueError(
                "Invalid path - unicode traversal pattern detected"
            )

        # Use werkzeug's safe_join for secure path joining
        # This handles path traversal attempts automatically. The base is
        # resolved the same way as the joined path below, so the
        # containment comparison is between two fully resolved paths.
        try:
            base_dir = resolve_allow_missing(Path(base_dir))
        except OSError:
            logger.warning("Path validation base directory is unresolvable")
            raise ValueError(
                "Invalid base directory - could not be resolved safely"
            )

        try:
            # safe_join returns None if the path tries to escape base_dir
            safe_path = safe_join(str(base_dir), user_input)
        except ValueError:
            raise
        except Exception as e:
            logger.warning(
                f"Path validation failed for input '{redact_and_bound_for_log(user_input, 200)}'"
            )
            raise ValueError(f"Invalid path: {e}") from e

        if safe_path is None:
            logger.warning(
                f"Path traversal attempt blocked: {redact_and_bound_for_log(user_input, 200)}"
            )
            raise ValueError("Invalid path - potential traversal attempt")

        result_path = Path(safe_path)

        # Filesystem-level containment (GHSA-gv87-qcw7-gjr8). safe_join is
        # a *string* operation: it rejects ``..`` and absolute inputs but
        # never touches the filesystem, so a symlinked directory or leaf
        # inside base_dir that points outside it is joined and returned
        # verbatim — the caller then reads through the link, outside the
        # base. Resolve the joined path and require its real location to
        # stay under the (already resolved) base. This is deliberately
        # NOT #3090's removed check: that one rejected *every* symlink
        # and broke legitimate layouts (bind mounts, macOS
        # /tmp -> /private/tmp); containment accepts any symlink whose
        # target resolves within base_dir and refuses only the escapes.
        # resolve_allow_missing() tolerates only a missing component, so
        # a not-yet-existing leaf is fine. Every other resolution error
        # raises and fails closed here: a symlink loop (ELOOP), a path
        # below a regular file (ENOTDIR), and ENAMETOOLONG. A bare
        # non-strict resolve() ignores ENAMETOOLONG (and, on 3.13+, a
        # loop) and appends the rest of the path unresolved, so a symlink
        # chain whose resolved form outgrows PATH_MAX could hide an escape.
        try:
            resolved_path = resolve_allow_missing(result_path)
        except OSError:
            logger.warning(
                f"Path resolution failed for input {redact_and_bound_for_log(user_input, 200)!r}"
            )
            raise ValueError("Invalid path - could not be resolved safely")
        if not resolved_path.is_relative_to(base_dir):
            logger.warning(
                f"Symlink containment violation blocked: {redact_and_bound_for_log(user_input, 200)!r}"
            )
            raise ValueError(
                "Invalid path - resolved path escapes base directory"
            )
        # Return the *lexical* (pre-resolve) path, not the resolved one:
        # the containment check has already run, and callers key external
        # behaviour off the requested name (e.g. FileResponse guesses the
        # content type from the served filename, and the storage manager
        # stores paths relative to the library root it passed in). Callers
        # that need the real location resolve it themselves.

        # Check extensions if required
        if (
            required_extensions
            and result_path.suffix not in required_extensions
        ):
            raise ValueError(
                f"Invalid file type. Allowed: {required_extensions}"
            )

        return result_path

    @staticmethod
    def validate_local_filesystem_path(
        user_path: str,
        restricted_dirs: Optional[list[Path]] = None,
    ) -> Path:
        """
        Validate a user-provided absolute filesystem path for local indexing.

        This is for features like local folder indexing where users need to
        access files anywhere on their own machine, but system directories
        should be blocked.

        Args:
            user_path: User-provided path string (absolute or with ~)
            restricted_dirs: List of restricted directories to block

        Returns:
            Validated and resolved Path object

        Raises:
            ValueError: If path is invalid or points to restricted location
        """
        import sys

        if not user_path or not isinstance(user_path, str):
            raise ValueError("Invalid path input")

        user_path = user_path.strip()

        # Basic sanitation: forbid null bytes and control characters
        if "\x00" in user_path:
            raise ValueError("Null bytes are not allowed in path")
        if any(ord(ch) < 32 for ch in user_path):
            raise ValueError("Control characters are not allowed in path")

        # Expand ~ to home directory
        if user_path.startswith("~"):
            home_dir = Path.home()
            relative_part = user_path[2:].lstrip("/")
            if relative_part:
                user_path = str(home_dir / relative_part)
            else:
                user_path = str(home_dir)

        # Disallow malformed Windows paths (e.g. "/C:/Windows")
        if (
            sys.platform == "win32"
            and user_path.startswith(("/", "\\"))
            and ":" in user_path
        ):
            raise ValueError("Malformed Windows path input")

        # Block path traversal patterns before resolving
        # This explicit check helps static analyzers understand the security intent
        if ".." in user_path:
            raise ValueError("Path traversal patterns not allowed")

        # Use safe_join to sanitize the path - this is recognized by static analyzers
        # For absolute paths, we validate against the root directory
        if user_path.startswith("/"):
            # Unix absolute path - use safe_join with root
            safe_path = safe_join("/", user_path.lstrip("/"))
            if safe_path is None:
                raise ValueError("Invalid path - failed security validation")
            validated_path = Path(safe_path).resolve()
        elif len(user_path) > 2 and user_path[1] == ":":
            # Windows absolute path (e.g., C:\Users\...)
            drive = user_path[:2]
            rest = user_path[2:].lstrip("\\").lstrip("/")
            safe_path = safe_join(drive + "\\", rest)
            if safe_path is None:
                raise ValueError("Invalid path - failed security validation")
            validated_path = Path(safe_path).resolve()
        else:
            # Relative path - resolve relative to current directory
            # Use safe_join to validate
            cwd = os.getcwd()
            safe_path = safe_join(cwd, user_path)
            if safe_path is None:
                raise ValueError("Invalid path - failed security validation")
            validated_path = Path(safe_path).resolve()

        # Default restricted directories
        if restricted_dirs is None:
            restricted_dirs = [
                Path("/etc"),
                Path("/sys"),
                Path("/proc"),
                Path("/dev"),
                Path("/root"),
                Path("/boot"),
                Path("/var/log"),
            ]
            # Add Windows system directories if on Windows
            if sys.platform == "win32":
                for drive in ["C:", "D:", "E:"]:
                    restricted_dirs.extend(
                        [
                            Path(f"{drive}\\Windows"),
                            Path(f"{drive}\\System32"),
                            Path(f"{drive}\\Program Files"),
                            Path(f"{drive}\\Program Files (x86)"),
                        ]
                    )

        # Check against restricted directories. Resolve each restricted dir so
        # the containment check holds on platforms where these are symlinks
        # (e.g. macOS /etc -> /private/etc, /var -> /private/var) — validated_path
        # is already resolved, so an unresolved "/etc" would never match
        # "/private/etc/...". resolve(strict=False) does not raise for a missing
        # or unreadable dir; fall back to the literal path if it ever does.
        for restricted in restricted_dirs:
            try:
                restricted_resolved = restricted.resolve()
            except OSError:
                restricted_resolved = restricted
            if validated_path.is_relative_to(restricted_resolved):
                # Log WHICH restricted dir was hit, not the user's submitted
                # path — a resolved local path can contain a username.
                logger.error(
                    f"Security: blocked access to a restricted directory ({restricted})"
                )
                raise ValueError("Cannot access system directories")

        return validated_path

    @staticmethod
    def sanitize_for_filesystem_ops(validated_path: Path) -> Path:
        """
        Re-sanitize a validated path for static analyzer recognition.

        This method takes an already-validated Path and passes it through
        werkzeug's safe_join to create a path that static analyzers like
        CodeQL recognize as sanitized.

        Note: This exists because CodeQL doesn't trace through custom validation
        functions. The path is already secure from validate_local_filesystem_path(),
        but safe_join makes that explicit to static analyzers.

        Args:
            validated_path: A Path that has already been validated by
                          validate_local_filesystem_path()

        Returns:
            A Path object safe for filesystem operations

        Raises:
            ValueError: If the path fails sanitization
        """
        if not validated_path.is_absolute():
            raise ValueError("Path must be absolute")

        # Use safe_join to create a sanitized path that static analyzers recognize
        # safe_join handles path traversal detection properly (not substring matching)
        path_str = str(validated_path)
        safe_path_str = safe_join("/", path_str.lstrip("/"))
        if safe_path_str is None:
            raise ValueError("Path failed security sanitization")

        return Path(safe_path_str)

    # A `confine_to_base()` helper once lived here — see #4868. It filtered a list
    # of paths down to those whose real (symlink-resolved) location stayed inside a
    # base directory, dropping symlink escapes and loops.

    @staticmethod
    def validate_model_path(
        model_path: str, model_root: Optional[str] = None
    ) -> Path:
        """
        Validate a model file path specifically.

        Args:
            model_path: Path to the model file
            model_root: Root directory for models (defaults to ~/.local/share/llm_models)

        Returns:
            Validated Path object

        Raises:
            ValueError: If the path is invalid
        """
        if model_root is None:
            # Default model root - uses centralized path config (respects LDR_DATA_DIR)
            model_root = str(get_models_directory())

        # Create model root if it doesn't exist. Lazy import: this sibling
        # module can be imported very early (e.g. from `security/__init__.py`
        # during package bootstrap), so import at call time rather than at
        # module top level to avoid any import-cycle risk.
        from .directory_creation import create_directory

        model_root_path = Path(model_root).resolve()
        create_directory(
            model_root_path,
            context="model root directory",
        )

        # Validate the path
        validated_path = PathValidator.validate_safe_path(
            model_path,
            model_root_path,
            allow_absolute=False,  # Models should always be relative to model root
            required_extensions=None,  # Models can have various extensions
        )

        if not validated_path:
            raise ValueError("Invalid model path")

        # Check if the file exists
        if not validated_path.exists():
            raise ValueError(f"Model file not found: {validated_path}")

        if not validated_path.is_file():
            raise ValueError(f"Model path is not a file: {validated_path}")

        return validated_path

    @staticmethod
    def validate_data_path(file_path: str, data_root: str) -> Path:
        """
        Validate a path within the data directory.

        Args:
            file_path: Path relative to data root
            data_root: The data root directory

        Returns:
            Validated Path object

        Raises:
            ValueError: If the path is invalid
        """
        validated_path = PathValidator.validate_safe_path(
            file_path,
            data_root,
            allow_absolute=False,  # Data paths should be relative
            required_extensions=None,
        )

        if not validated_path:
            raise ValueError("Invalid data path")

        return validated_path

    @staticmethod
    def validate_config_path(
        config_path: str, config_root: Optional[Union[str, Path]] = None
    ) -> Path:
        """
        Validate a configuration file path.

        Args:
            config_path: Path to config file
            config_root: Root directory for configs (optional for absolute paths)

        Returns:
            Validated Path object

        Raises:
            ValueError: If the path is invalid
        """
        # Validate input: reject null bytes, then normalize whitespace
        if not config_path or not isinstance(config_path, str):
            raise ValueError("Invalid config path input")

        if "\x00" in config_path:
            raise ValueError("Null bytes are not allowed in config path")

        config_path = config_path.strip()

        # Check for path traversal attempts in the string itself
        # Define restricted system directories that should never be accessed
        RESTRICTED_PREFIXES = ("etc", "proc", "sys", "dev")

        if ".." in config_path:
            raise ValueError("Invalid path - potential traversal attempt")

        # Check if path starts with any restricted system directory
        normalized_path = config_path.lstrip("/").lower()
        for restricted in RESTRICTED_PREFIXES:
            if (
                normalized_path.startswith(restricted + "/")
                or normalized_path == restricted
            ):
                raise ValueError(
                    f"Invalid path - restricted system directory: {restricted}"
                )

        # For config files, we might allow absolute paths with restrictions
        # Check if path starts with / or drive letter (Windows) to detect absolute paths
        # This avoids using Path() or os.path on user input
        is_absolute = (
            config_path.startswith("/")  # Unix absolute
            or (
                len(config_path) > 2 and config_path[1] == ":"
            )  # Windows absolute
        )

        if is_absolute:
            # For absolute paths, use safe_join with root directory
            # This validates the path without using Path() directly on user input
            # Use safe_join to validate the absolute path
            safe_path = safe_join("/", config_path)
            if safe_path is None:
                raise ValueError("Invalid absolute path")

            # Now it's safe to create Path object from validated string
            path_obj = Path(safe_path)

            # Additional validation for config files
            if path_obj.suffix not in PathValidator.CONFIG_EXTENSIONS:
                raise ValueError(f"Invalid config file type: {path_obj.suffix}")

            # Check existence using validated path
            if not path_obj.exists():
                raise ValueError(f"Config file not found: {path_obj}")

            return path_obj
        # For relative paths, use the config root
        if config_root is None:
            from ..config.paths import get_data_directory

            config_root = get_data_directory()

        validated = PathValidator.validate_safe_path(
            config_path,
            config_root,
            allow_absolute=False,
            required_extensions=PathValidator.CONFIG_EXTENSIONS,
        )
        if validated is None:
            raise ValueError(f"Invalid config path: {config_path}")
        return validated