~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Centralized path validation utilities for security.
This module provides secure path validation to prevent path traversal attacks
and other filesystem-based security vulnerabilities.
"""
import errno
import ntpath
import os
import posixpath
import re
import unicodedata
from pathlib import Path
from typing import Optional, Union
from urllib.parse import unquote
from loguru import logger
from werkzeug.security import safe_join
from ..config.paths import get_models_directory
from .log_sanitizer import redact_and_bound_for_log
# Encoded forms of ".." and "." that path traversal attacks use to bypass
# naive `..`-substring checks. The list covers both single and double URL
# encoding (where the % itself is encoded as %25).
_ENCODED_TRAVERSAL_TOKENS = (
"%2e%2e", # ..
"%2E%2E",
"%2e.", # .. (mixed)
".%2e",
"%252e%252e", # double-encoded ..
"%252E%252E",
"%2f", # /
"%2F",
)
def _has_encoded_traversal(text: str) -> bool:
"""Return True if `text` contains URL-encoded path-traversal tokens.
Checks for both single and double URL encoding of '..', '.', and '/'.
Decoders downstream may convert these into real '..' segments that
escape the base directory; reject the input before that happens.
"""
lowered = text.lower()
for tok in _ENCODED_TRAVERSAL_TOKENS:
if tok.lower() in lowered:
return True
# Recursive decode: if unquote changes the string, the original
# contained encoded characters. Re-check the decoded form for ".."
# in case the encoding used is one not enumerated above.
decoded = unquote(text)
if decoded != text and ".." in decoded:
return True
# Double-decode catches %252e%252e -> %2e%2e -> ..
double_decoded = unquote(decoded)
if double_decoded != decoded and ".." in double_decoded:
return True
return False
def _has_unicode_traversal(text: str) -> bool:
"""Return True if `text` contains unicode look-alikes for path traversal.
Many unicode characters NFKC-normalize to '.' or '/', so an attacker
can use full-width periods (U+FF0E '.') or other look-alikes to
bypass naive '..'-substring checks. After normalization, '..' or
'/..' segments indicate an attempt to traverse.
"""
normalized = unicodedata.normalize("NFKC", text)
if normalized == text:
return False
return (
".." in normalized
or normalized.startswith("/..")
or "/.." in normalized
)
# ``os.path.ALLOW_MISSING`` exists on Python 3.14 and was backported to
# 3.12.11 and 3.13.4. Older patch releases use the fallback below. It is
# read from the platform's path module, picked the way ``os`` picks
# ``os.path``, because the check-pathlib-usage hook forbids ``os.path``
# and the sentinel has no pathlib spelling.
_PLATFORM_PATH_MODULE = ntpath if os.name == "nt" else posixpath
_ALLOW_MISSING = getattr(_PLATFORM_PATH_MODULE, "ALLOW_MISSING", None)
def resolve_allow_missing(path: Union[str, Path]) -> Path:
"""Resolve every symlink in ``path``. Only a missing tail is tolerated.
Use this, never a bare ``Path.resolve()``, wherever a resolved path is
compared against a containment boundary. Non-strict ``resolve()``
ignores every ``OSError`` that ``lstat`` raises, not only "does not
exist". A short in-base symlink to a deep in-base directory chain makes
the resolved prefix longer than PATH_MAX, so every later ``lstat`` fails
with ENAMETOOLONG. The remaining components, including a symlink that
leaves the base, are then appended without being resolved. The result
looks contained, but the kernel resolves the short path without that
limit and follows the link out (the CVE-2025-4517 class).
Here a component that does not exist is appended as written, so a leaf
that has not been created yet still resolves. Any other failure raises
``OSError``: ENAMETOOLONG, ELOOP (a symlink loop), EACCES, and ENOTDIR
(a path that continues below a regular file). Python 3.12 reports a
loop as ``RuntimeError``, which is re-raised as ``OSError(ELOOP)``.
Without ``os.path.ALLOW_MISSING`` the longest prefix whose last entry
exists is resolved strictly and the missing tail is appended. A dangling
symlink, or a ``..`` in the missing tail, raises there as well. Both
are refusals that the ``ALLOW_MISSING`` path would not make, so the
fallback is only ever stricter.
"""
if isinstance(path, str):
path = Path(path)
try:
if _ALLOW_MISSING is not None:
return path.resolve(strict=_ALLOW_MISSING)
return _resolve_allow_missing_fallback(path)
except RuntimeError as exc:
raise OSError(errno.ELOOP, "Symlink loop", str(path)) from exc
def _resolve_allow_missing_fallback(path: Path) -> Path:
"""``resolve_allow_missing`` for a Python without ``ALLOW_MISSING``."""
path = path.absolute()
missing: list[str] = []
head = path
while True:
try:
# lstat, not exists(): only ENOENT means "missing". Every
# other error (ENAMETOOLONG, ENOTDIR, EACCES) propagates.
os.lstat(head)
break
except FileNotFoundError:
if head.parent == head:
raise
missing.append(head.name)
head = head.parent
if ".." in missing:
# ".." below a missing component cannot be resolved against the
# filesystem, and is_relative_to() would compare it lexically.
raise OSError(errno.ENOENT, "Unresolvable '..' segment", str(path))
# strict=True raises for a dangling symlink, a loop and ENAMETOOLONG.
return head.resolve(strict=True).joinpath(*reversed(missing))
class PathValidator:
"""Centralized path validation for security."""
# Regex for safe filename/path characters
SAFE_PATH_PATTERN = re.compile(r"^[a-zA-Z0-9._/\-]+$")
# Allowed config file extensions
CONFIG_EXTENSIONS = (".json", ".yaml", ".yml", ".toml", ".ini", ".conf")
@staticmethod
def validate_safe_path(
user_input: str,
base_dir: Union[str, Path],
allow_absolute: bool = False,
required_extensions: Optional[tuple] = None,
) -> Optional[Path]:
"""
Validate and sanitize a user-provided path.
Args:
user_input: The user-provided path string
base_dir: The safe base directory to contain paths within
allow_absolute: Whether to allow absolute paths (with restrictions)
required_extensions: Tuple of required file extensions (e.g., ('.json', '.yaml'))
Returns:
The validated Path. It is never None: every rejection raises.
Raises:
ValueError: If the path is invalid or unsafe
"""
if not user_input or not isinstance(user_input, str):
raise ValueError("Invalid path input")
if "\x00" in user_input:
raise ValueError("Null bytes are not allowed in path")
# Strip whitespace
user_input = user_input.strip()
# (#6938: the input is a client-chosen path — e.g. the
# unauthenticated /static route — so the rejection log lines
# below interpolate a redacted, bounded form, never the raw value.
# It is built only in those rejection branches: this runs on every
# static-file lookup, and redaction costs up to tens of
# milliseconds on a long adversarial path.)
# Reject URL-encoded traversal tokens (single or double encoded).
# safe_join's literal-".."-check doesn't catch %2e%2e or %252e%252e,
# so do it explicitly here before any decoding takes place.
if _has_encoded_traversal(user_input):
logger.warning(
f"Encoded path-traversal attempt blocked: {redact_and_bound_for_log(user_input, 200)!r}"
)
raise ValueError(
"Invalid path - encoded traversal pattern detected"
)
# Reject unicode look-alike traversal (e.g. full-width '..').
# NFKC normalization is the same form most filesystems / browsers
# apply; if normalizing produces a '..' segment the original was a
# disguised traversal attempt.
if _has_unicode_traversal(user_input):
logger.warning(
f"Unicode path-traversal attempt blocked: {redact_and_bound_for_log(user_input, 200)!r}"
)
raise ValueError(
"Invalid path - unicode traversal pattern detected"
)
# Use werkzeug's safe_join for secure path joining
# This handles path traversal attempts automatically. The base is
# resolved the same way as the joined path below, so the
# containment comparison is between two fully resolved paths.
try:
base_dir = resolve_allow_missing(Path(base_dir))
except OSError:
logger.warning("Path validation base directory is unresolvable")
raise ValueError(
"Invalid base directory - could not be resolved safely"
)
try:
# safe_join returns None if the path tries to escape base_dir
safe_path = safe_join(str(base_dir), user_input)
except ValueError:
raise
except Exception as e:
logger.warning(
f"Path validation failed for input '{redact_and_bound_for_log(user_input, 200)}'"
)
raise ValueError(f"Invalid path: {e}") from e
if safe_path is None:
logger.warning(
f"Path traversal attempt blocked: {redact_and_bound_for_log(user_input, 200)}"
)
raise ValueError("Invalid path - potential traversal attempt")
result_path = Path(safe_path)
# Filesystem-level containment (GHSA-gv87-qcw7-gjr8). safe_join is
# a *string* operation: it rejects ``..`` and absolute inputs but
# never touches the filesystem, so a symlinked directory or leaf
# inside base_dir that points outside it is joined and returned
# verbatim — the caller then reads through the link, outside the
# base. Resolve the joined path and require its real location to
# stay under the (already resolved) base. This is deliberately
# NOT #3090's removed check: that one rejected *every* symlink
# and broke legitimate layouts (bind mounts, macOS
# /tmp -> /private/tmp); containment accepts any symlink whose
# target resolves within base_dir and refuses only the escapes.
# resolve_allow_missing() tolerates only a missing component, so
# a not-yet-existing leaf is fine. Every other resolution error
# raises and fails closed here: a symlink loop (ELOOP), a path
# below a regular file (ENOTDIR), and ENAMETOOLONG. A bare
# non-strict resolve() ignores ENAMETOOLONG (and, on 3.13+, a
# loop) and appends the rest of the path unresolved, so a symlink
# chain whose resolved form outgrows PATH_MAX could hide an escape.
try:
resolved_path = resolve_allow_missing(result_path)
except OSError:
logger.warning(
f"Path resolution failed for input {redact_and_bound_for_log(user_input, 200)!r}"
)
raise ValueError("Invalid path - could not be resolved safely")
if not resolved_path.is_relative_to(base_dir):
logger.warning(
f"Symlink containment violation blocked: {redact_and_bound_for_log(user_input, 200)!r}"
)
raise ValueError(
"Invalid path - resolved path escapes base directory"
)
# Return the *lexical* (pre-resolve) path, not the resolved one:
# the containment check has already run, and callers key external
# behaviour off the requested name (e.g. FileResponse guesses the
# content type from the served filename, and the storage manager
# stores paths relative to the library root it passed in). Callers
# that need the real location resolve it themselves.
# Check extensions if required
if (
required_extensions
and result_path.suffix not in required_extensions
):
raise ValueError(
f"Invalid file type. Allowed: {required_extensions}"
)
return result_path
@staticmethod
def validate_local_filesystem_path(
user_path: str,
restricted_dirs: Optional[list[Path]] = None,
) -> Path:
"""
Validate a user-provided absolute filesystem path for local indexing.
This is for features like local folder indexing where users need to
access files anywhere on their own machine, but system directories
should be blocked.
Args:
user_path: User-provided path string (absolute or with ~)
restricted_dirs: List of restricted directories to block
Returns:
Validated and resolved Path object
Raises:
ValueError: If path is invalid or points to restricted location
"""
import sys
if not user_path or not isinstance(user_path, str):
raise ValueError("Invalid path input")
user_path = user_path.strip()
# Basic sanitation: forbid null bytes and control characters
if "\x00" in user_path:
raise ValueError("Null bytes are not allowed in path")
if any(ord(ch) < 32 for ch in user_path):
raise ValueError("Control characters are not allowed in path")
# Expand ~ to home directory
if user_path.startswith("~"):
home_dir = Path.home()
relative_part = user_path[2:].lstrip("/")
if relative_part:
user_path = str(home_dir / relative_part)
else:
user_path = str(home_dir)
# Disallow malformed Windows paths (e.g. "/C:/Windows")
if (
sys.platform == "win32"
and user_path.startswith(("/", "\\"))
and ":" in user_path
):
raise ValueError("Malformed Windows path input")
# Block path traversal patterns before resolving
# This explicit check helps static analyzers understand the security intent
if ".." in user_path:
raise ValueError("Path traversal patterns not allowed")
# Use safe_join to sanitize the path - this is recognized by static analyzers
# For absolute paths, we validate against the root directory
if user_path.startswith("/"):
# Unix absolute path - use safe_join with root
safe_path = safe_join("/", user_path.lstrip("/"))
if safe_path is None:
raise ValueError("Invalid path - failed security validation")
validated_path = Path(safe_path).resolve()
elif len(user_path) > 2 and user_path[1] == ":":
# Windows absolute path (e.g., C:\Users\...)
drive = user_path[:2]
rest = user_path[2:].lstrip("\\").lstrip("/")
safe_path = safe_join(drive + "\\", rest)
if safe_path is None:
raise ValueError("Invalid path - failed security validation")
validated_path = Path(safe_path).resolve()
else:
# Relative path - resolve relative to current directory
# Use safe_join to validate
cwd = os.getcwd()
safe_path = safe_join(cwd, user_path)
if safe_path is None:
raise ValueError("Invalid path - failed security validation")
validated_path = Path(safe_path).resolve()
# Default restricted directories
if restricted_dirs is None:
restricted_dirs = [
Path("/etc"),
Path("/sys"),
Path("/proc"),
Path("/dev"),
Path("/root"),
Path("/boot"),
Path("/var/log"),
]
# Add Windows system directories if on Windows
if sys.platform == "win32":
for drive in ["C:", "D:", "E:"]:
restricted_dirs.extend(
[
Path(f"{drive}\\Windows"),
Path(f"{drive}\\System32"),
Path(f"{drive}\\Program Files"),
Path(f"{drive}\\Program Files (x86)"),
]
)
# Check against restricted directories. Resolve each restricted dir so
# the containment check holds on platforms where these are symlinks
# (e.g. macOS /etc -> /private/etc, /var -> /private/var) — validated_path
# is already resolved, so an unresolved "/etc" would never match
# "/private/etc/...". resolve(strict=False) does not raise for a missing
# or unreadable dir; fall back to the literal path if it ever does.
for restricted in restricted_dirs:
try:
restricted_resolved = restricted.resolve()
except OSError:
restricted_resolved = restricted
if validated_path.is_relative_to(restricted_resolved):
# Log WHICH restricted dir was hit, not the user's submitted
# path — a resolved local path can contain a username.
logger.error(
f"Security: blocked access to a restricted directory ({restricted})"
)
raise ValueError("Cannot access system directories")
return validated_path
@staticmethod
def sanitize_for_filesystem_ops(validated_path: Path) -> Path:
"""
Re-sanitize a validated path for static analyzer recognition.
This method takes an already-validated Path and passes it through
werkzeug's safe_join to create a path that static analyzers like
CodeQL recognize as sanitized.
Note: This exists because CodeQL doesn't trace through custom validation
functions. The path is already secure from validate_local_filesystem_path(),
but safe_join makes that explicit to static analyzers.
Args:
validated_path: A Path that has already been validated by
validate_local_filesystem_path()
Returns:
A Path object safe for filesystem operations
Raises:
ValueError: If the path fails sanitization
"""
if not validated_path.is_absolute():
raise ValueError("Path must be absolute")
# Use safe_join to create a sanitized path that static analyzers recognize
# safe_join handles path traversal detection properly (not substring matching)
path_str = str(validated_path)
safe_path_str = safe_join("/", path_str.lstrip("/"))
if safe_path_str is None:
raise ValueError("Path failed security sanitization")
return Path(safe_path_str)
# A `confine_to_base()` helper once lived here — see #4868. It filtered a list
# of paths down to those whose real (symlink-resolved) location stayed inside a
# base directory, dropping symlink escapes and loops.
@staticmethod
def validate_model_path(
model_path: str, model_root: Optional[str] = None
) -> Path:
"""
Validate a model file path specifically.
Args:
model_path: Path to the model file
model_root: Root directory for models (defaults to ~/.local/share/llm_models)
Returns:
Validated Path object
Raises:
ValueError: If the path is invalid
"""
if model_root is None:
# Default model root - uses centralized path config (respects LDR_DATA_DIR)
model_root = str(get_models_directory())
# Create model root if it doesn't exist. Lazy import: this sibling
# module can be imported very early (e.g. from `security/__init__.py`
# during package bootstrap), so import at call time rather than at
# module top level to avoid any import-cycle risk.
from .directory_creation import create_directory
model_root_path = Path(model_root).resolve()
create_directory(
model_root_path,
context="model root directory",
)
# Validate the path
validated_path = PathValidator.validate_safe_path(
model_path,
model_root_path,
allow_absolute=False, # Models should always be relative to model root
required_extensions=None, # Models can have various extensions
)
if not validated_path:
raise ValueError("Invalid model path")
# Check if the file exists
if not validated_path.exists():
raise ValueError(f"Model file not found: {validated_path}")
if not validated_path.is_file():
raise ValueError(f"Model path is not a file: {validated_path}")
return validated_path
@staticmethod
def validate_data_path(file_path: str, data_root: str) -> Path:
"""
Validate a path within the data directory.
Args:
file_path: Path relative to data root
data_root: The data root directory
Returns:
Validated Path object
Raises:
ValueError: If the path is invalid
"""
validated_path = PathValidator.validate_safe_path(
file_path,
data_root,
allow_absolute=False, # Data paths should be relative
required_extensions=None,
)
if not validated_path:
raise ValueError("Invalid data path")
return validated_path
@staticmethod
def validate_config_path(
config_path: str, config_root: Optional[Union[str, Path]] = None
) -> Path:
"""
Validate a configuration file path.
Args:
config_path: Path to config file
config_root: Root directory for configs (optional for absolute paths)
Returns:
Validated Path object
Raises:
ValueError: If the path is invalid
"""
# Validate input: reject null bytes, then normalize whitespace
if not config_path or not isinstance(config_path, str):
raise ValueError("Invalid config path input")
if "\x00" in config_path:
raise ValueError("Null bytes are not allowed in config path")
config_path = config_path.strip()
# Check for path traversal attempts in the string itself
# Define restricted system directories that should never be accessed
RESTRICTED_PREFIXES = ("etc", "proc", "sys", "dev")
if ".." in config_path:
raise ValueError("Invalid path - potential traversal attempt")
# Check if path starts with any restricted system directory
normalized_path = config_path.lstrip("/").lower()
for restricted in RESTRICTED_PREFIXES:
if (
normalized_path.startswith(restricted + "/")
or normalized_path == restricted
):
raise ValueError(
f"Invalid path - restricted system directory: {restricted}"
)
# For config files, we might allow absolute paths with restrictions
# Check if path starts with / or drive letter (Windows) to detect absolute paths
# This avoids using Path() or os.path on user input
is_absolute = (
config_path.startswith("/") # Unix absolute
or (
len(config_path) > 2 and config_path[1] == ":"
) # Windows absolute
)
if is_absolute:
# For absolute paths, use safe_join with root directory
# This validates the path without using Path() directly on user input
# Use safe_join to validate the absolute path
safe_path = safe_join("/", config_path)
if safe_path is None:
raise ValueError("Invalid absolute path")
# Now it's safe to create Path object from validated string
path_obj = Path(safe_path)
# Additional validation for config files
if path_obj.suffix not in PathValidator.CONFIG_EXTENSIONS:
raise ValueError(f"Invalid config file type: {path_obj.suffix}")
# Check existence using validated path
if not path_obj.exists():
raise ValueError(f"Config file not found: {path_obj}")
return path_obj
# For relative paths, use the config root
if config_root is None:
from ..config.paths import get_data_directory
config_root = get_data_directory()
validated = PathValidator.validate_safe_path(
config_path,
config_root,
allow_absolute=False,
required_extensions=PathValidator.CONFIG_EXTENSIONS,
)
if validated is None:
raise ValueError(f"Invalid config path: {config_path}")
return validated