Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

19.7 KB
Raw Blame
"""
Centralized URL validation utilities for security.

This module provides secure URL validation to prevent XSS attacks,
data exfiltration, and other URL-based security vulnerabilities.
"""

import re
from typing import Optional, List
from urllib.parse import urlparse, urljoin, unquote
from loguru import logger

from .log_sanitizer import redact_and_bound_for_log


class URLValidationError(ValueError):
    """Raised when URL construction or validation fails."""

    pass


class URLValidator:
    """Centralized URL validation for security."""

    # Unsafe URL schemes that could lead to XSS or data exfiltration
    UNSAFE_SCHEMES = (
        "javascript",
        "data",
        "vbscript",
        "about",
        "blob",
        "file",
    )

    # Safe schemes for external links
    SAFE_SCHEMES = ("http", "https", "ftp", "ftps")

    # Email scheme
    EMAIL_SCHEME = "mailto"

    # Common academic/research domains that should be allowed
    TRUSTED_ACADEMIC_DOMAINS = (
        "arxiv.org",
        "pubmed.ncbi.nlm.nih.gov",
        "ncbi.nlm.nih.gov",
        "biorxiv.org",
        "medrxiv.org",
        "doi.org",
        "nature.com",
        "science.org",
        "sciencedirect.com",
        "springer.com",
        "wiley.com",
        "plos.org",
        "pnas.org",
        "ieee.org",
        "acm.org",
    )

    @staticmethod
    def is_unsafe_scheme(url: str) -> bool:
        """
        Check if a URL uses an unsafe scheme.

        Args:
            url: The URL to check

        Returns:
            True if the URL uses an unsafe scheme, False otherwise
        """
        if not url:
            return False

        # Normalize the URL - trim whitespace and convert to lowercase
        normalized_url = url.strip().lower()

        # Check for unsafe schemes
        for scheme in URLValidator.UNSAFE_SCHEMES:
            if normalized_url.startswith(f"{scheme}:"):
                logger.warning(
                    "Unsafe URL scheme detected: {} in URL: {}",
                    scheme,
                    # (#6938: redacted before it is cut, so the cut cannot
                    # drop the ``@`` after URL userinfo.)
                    redact_and_bound_for_log(url, 100),
                )
                return True

        return False

    @staticmethod
    def is_safe_url(
        url: str,
        require_scheme: bool = True,
        allow_fragments: bool = True,
        allow_mailto: bool = False,
        trusted_domains: Optional[List[str]] = None,
    ) -> bool:
        """
        Validate if a URL is safe to use.

        Args:
            url: The URL to validate
            require_scheme: Whether to require an explicit scheme
            allow_fragments: Whether to allow fragment identifiers (#)
            allow_mailto: Whether to allow mailto: links
            trusted_domains: Optional list of trusted domains

        Returns:
            True if the URL is safe, False otherwise
        """
        if not url or not isinstance(url, str):
            return False

        # Check for unsafe schemes first
        if URLValidator.is_unsafe_scheme(url):
            return False

        # Handle fragment-only URLs
        if url.startswith("#"):
            return allow_fragments

        # Parse the URL
        try:
            parsed = urlparse(url)
        except Exception:
            logger.warning(
                "Failed to parse URL '{}'",
                redact_and_bound_for_log(str(url), 100),
            )
            return False

        # Check scheme
        if not parsed.scheme:
            if require_scheme:
                return False
            # If no scheme is required, assume http/https for URL parsing
            parsed = urlparse(f"http://{url}")  # DevSkim: ignore DS137138

        scheme_lower = parsed.scheme.lower()

        # Check if it's a mailto link
        if scheme_lower == URLValidator.EMAIL_SCHEME:
            return allow_mailto

        # Check if it's a safe scheme
        if scheme_lower not in URLValidator.SAFE_SCHEMES:
            logger.warning(f"Unsafe URL scheme: {scheme_lower}")
            return False

        # Validate domain if trusted domains are specified
        if trusted_domains and parsed.hostname:
            hostname_lower = parsed.hostname.lower()
            if not any(
                hostname_lower == domain.lower()
                or hostname_lower.endswith(f".{domain.lower()}")
                for domain in trusted_domains
            ):
                logger.warning(
                    f"URL domain not in trusted list: {parsed.hostname}"
                )
                return False

        # Check for suspicious patterns in the URL
        if URLValidator._has_suspicious_patterns(url):
            return False

        return True

    @staticmethod
    def _has_suspicious_patterns(url: str) -> bool:
        """
        Check for suspicious patterns in URLs that might indicate attacks.

        Args:
            url: The URL to check

        Returns:
            True if suspicious patterns are found, False otherwise
        """
        suspicious_patterns = [
            # Double encoding
            r"%25[0-9a-fA-F]{2}",
            # Null bytes
            r"%00",
            # Unicode encoding bypass attempts
            r"\\u[0-9a-fA-F]{4}",
            # HTML entity encoding
            r"&(#x?[0-9a-fA-F]+|[a-zA-Z]+);",
        ]

        for pattern in suspicious_patterns:
            if re.search(pattern, url, re.IGNORECASE):
                logger.warning(f"Suspicious pattern found in URL: {pattern}")
                return True

        return False

    @staticmethod
    def sanitize_url(url: str, default_scheme: str = "https") -> Optional[str]:
        """
        Sanitize a URL by adding a scheme if missing and validating it.

        Args:
            url: The URL to sanitize
            default_scheme: The default scheme to add if missing

        Returns:
            Sanitized URL or None if the URL is unsafe
        """
        if not url:
            return None

        # Check for unsafe schemes
        if URLValidator.is_unsafe_scheme(url):
            return None

        # Strip whitespace
        url = url.strip()

        # Parse the URL
        try:
            parsed = urlparse(url)

            # Add scheme if missing
            if not parsed.scheme:
                url = f"{default_scheme}://{url}"
                parsed = urlparse(url)

            # Validate the final URL
            if URLValidator.is_safe_url(url, require_scheme=True):
                return url

        except Exception:
            logger.warning(
                "Failed to sanitize URL '{}'",
                redact_and_bound_for_log(str(url), 100),
            )

        return None

    @staticmethod
    def is_academic_url(url: str) -> bool:
        """
        Check if a URL is from a known academic/research domain.

        Args:
            url: The URL to check

        Returns:
            True if the URL is from an academic domain, False otherwise
        """
        try:
            parsed = urlparse(url)
            if parsed.hostname:
                hostname_lower = parsed.hostname.lower()
                return any(
                    hostname_lower == domain
                    or hostname_lower.endswith(f".{domain}")
                    for domain in URLValidator.TRUSTED_ACADEMIC_DOMAINS
                )
        except Exception:
            logger.debug(
                "URL parsing may fail on malformed input; treat as non-academic",
                exc_info=True,
            )

        return False

    @staticmethod
    def extract_doi(url: str) -> Optional[str]:
        """
        Extract DOI from a URL if present.

        Args:
            url: The URL to extract DOI from

        Returns:
            The DOI if found, None otherwise
        """
        # Common DOI patterns with explicit pattern identification
        doi_patterns = [
            (
                r"10\.\d{4,}(?:\.\d+)*\/[-._;()\/:a-zA-Z0-9]+",
                0,
            ),  # Direct DOI, group 0
            (r"doi\.org\/(10\.\d{4,}[^\s]*)", 1),  # doi.org URL, group 1
        ]

        for pattern, group_index in doi_patterns:
            match = re.search(pattern, url, re.IGNORECASE)
            if match:
                return match.group(group_index)

        return None

    @staticmethod
    def is_safe_http_link(url: object) -> bool:
        """Allow absolute HTTP(S) links without interpreting path/query data.

        Literal percent signs, entity-like query values and trailing-dot hosts
        are valid in browser links. Callback-specific suspicious-pattern checks
        would reject them. Callers must still HTML-escape the URL in a quoted
        attribute; this policy does not authorize server-side requests.
        """
        if not isinstance(url, str) or not url:
            return False

        # Browsers and URL parsers can strip raw control characters. Reject
        # those ambiguous inputs while preserving percent-encoded URL data.
        if any(ord(char) < 0x20 or ord(char) == 0x7F for char in url):
            return False

        try:
            parsed = urlparse(url)
            # Accessing port also validates its numeric form and range.
            port = parsed.port
            return (
                parsed.scheme in ("http", "https")
                and bool(parsed.hostname)
                and (port is None or 0 <= port <= 65535)
            )
        except ValueError:
            return False

    @staticmethod
    def validate_http_url(url: str) -> bool:
        """
        Validate that a callback URL is well-formed and safe for HTTP/HTTPS use.

        This is stricter than is_safe_url() and specifically validates HTTP/HTTPS
        URLs for use as application callbacks (e.g., in notifications, redirects).
        It does NOT validate Apprise service URLs which use other protocols.

        Args:
            url: HTTP/HTTPS callback URL to validate

        Returns:
            True if valid

        Raises:
            URLValidationError: If URL is invalid
        """
        if not url or not isinstance(url, str):
            raise URLValidationError("URL must be a non-empty string")

        try:
            parsed = urlparse(url)
        except URLValidationError:
            raise
        except Exception as e:
            raise URLValidationError(f"Failed to validate URL: {e}") from e

        # Must have a scheme
        if not parsed.scheme:
            raise URLValidationError("URL must have a scheme (http or https)")

        # Must be http or https (callback URLs only)
        if parsed.scheme not in ("http", "https"):
            raise URLValidationError(
                f"URL scheme must be http or https, got: {parsed.scheme}"
            )

        # Use the general security validator for additional safety
        if not URLValidator.is_safe_url(url, require_scheme=True):
            raise URLValidationError(f"URL failed security validation: {url}")

        # Must have a netloc (hostname)
        if not parsed.netloc:
            raise URLValidationError("URL must have a hostname")

        # Check for obvious hostname issues
        if parsed.netloc.startswith(".") or parsed.netloc.endswith("."):
            raise URLValidationError(f"Invalid hostname: {parsed.netloc}")

        # Path should be valid if present
        if parsed.path and not parsed.path.startswith("/"):
            raise URLValidationError(
                f"URL path must start with /: {parsed.path}"
            )

        return True

    @staticmethod
    def is_safe_redirect_url(target: str, host_url: str) -> bool:
        """
        Validate that a redirect target is safe (same host).

        Prevents open redirect attacks by ensuring the target URL
        is either relative or points to the same host as the application.
        Also prevents CRLF injection attacks by rejecting URLs containing
        carriage return or line feed characters.

        Uses the standard Flask pattern from:
        https://github.com/fengsp/flask-snippets/blob/master/security/redirect_back.py

        Security protections implemented:
        - CRLF injection prevention
        - Protocol-relative URL bypass (//evil.com)
        - Triple-slash bypass (///evil.com)
        - URL-encoded bypass attempts
        - Backslash bypass (treated as forward slash by some browsers)
        - Path traversal detection

        Args:
            target: The redirect URL to validate (can be relative or absolute)
            host_url: The application's host URL (e.g., request.host_url)

        Returns:
            True if the URL is safe to redirect to, False otherwise
        """
        if not target:
            return False

        # Prevent CRLF injection by rejecting URLs with CR or LF characters
        if "\r" in target or "\n" in target:
            logger.warning("CRLF injection attempt detected in redirect URL")
            return False

        # Normalize and decode URL to detect encoded bypass attempts
        # This catches %2f%2f (encoded //) and similar tricks
        decoded_target = unquote(target)

        # Check for encoded CRLF injection (e.g. %0d%0a)
        if "\r" in decoded_target or "\n" in decoded_target:
            logger.warning(
                "Encoded CRLF injection attempt detected in redirect URL"
            )
            return False

        # Check for backslash (treated as forward slash by some browsers)
        # Prevents \evil.com and \\evil.com bypasses
        if "\\" in decoded_target:
            logger.warning("Backslash detected in redirect URL")
            return False

        # Check for protocol-relative URLs before stripping slashes
        # //evil.com would be treated as absolute URL by urljoin()
        if decoded_target.startswith("//"):
            logger.warning("Protocol-relative URL detected in redirect")
            return False

        # Block null bytes (truncation attacks in some URL processors)
        if "\x00" in decoded_target:
            logger.warning("Null byte detected in redirect URL")
            return False

        # Block path traversal patterns (/../, /.. at end, ../ at start)
        if re.search(r"(^|/)\.\.(/|$|\?|#)", decoded_target):
            logger.warning(
                "Path traversal detected in redirect URL: {}",
                # (#6938: the ``next`` target is a raw query param.)
                redact_and_bound_for_log(target, 200),
            )
            return False

        ref_url = urlparse(host_url)
        test_url = urlparse(urljoin(host_url, target))

        if test_url.path.startswith("//"):
            logger.warning(
                "Double-slash path in redirect URL: {}",
                redact_and_bound_for_log(target, 200),
            )
            return False

        return (
            test_url.scheme in ("http", "https")
            and ref_url.netloc == test_url.netloc
        )

    @staticmethod
    def get_safe_redirect_path(target: str, host_url: str) -> Optional[str]:
        """
        Validate a redirect target and return its path-only form.

        Combines is_safe_redirect_url validation with defense-in-depth path
        extraction. The target is resolved against host_url via urljoin, then
        only the path, query, and fragment are returned. This ensures that
        even a hypothetical validator bypass cannot cause an external redirect,
        since path-only URLs always resolve to the same host.

        Args:
            target: The redirect URL to validate (can be relative or absolute)
            host_url: The application's host URL (e.g., request.host_url)

        Returns:
            The safe path string (e.g., "/dashboard?tab=settings") if the
            target is valid, or None if the target is unsafe.
        """
        if not URLValidator.is_safe_redirect_url(target, host_url):
            return None

        resolved = urljoin(host_url, target)
        parsed = urlparse(resolved)
        safe_path = parsed.path or "/"
        if parsed.query:
            safe_path += "?" + parsed.query
        if parsed.fragment:
            safe_path += "#" + parsed.fragment

        # Final assertion: the constructed path must be relative (no netloc).
        # This is always true by construction but guards against future bugs.
        if urlparse(safe_path).netloc:
            return None
        return safe_path


def get_javascript_url_validator() -> str:
    """
    Get JavaScript code for URL validation that matches the Python implementation.

    Returns:
        JavaScript code as a string that can be embedded in web pages
    """
    return r"""
    // URL validation utilities matching Python URLValidator
    const URLValidator = {
        UNSAFE_SCHEMES: ['javascript', 'data', 'vbscript', 'about', 'blob', 'file'],
        SAFE_SCHEMES: ['http', 'https', 'ftp', 'ftps'],
        EMAIL_SCHEME: 'mailto',

        isUnsafeScheme: function(url) {
            if (!url) return false;

            const normalizedUrl = url.trim().toLowerCase();

            for (const scheme of this.UNSAFE_SCHEMES) {
                if (normalizedUrl.startsWith(scheme + ':')) {
                    console.warn(`Unsafe URL scheme detected: ${scheme}`);
                    return true;
                }
            }

            return false;
        },

        isSafeUrl: function(url, options = {}) {
            const {
                requireScheme = true,
                allowFragments = true,
                allowMailto = false,
                trustedDomains = []
            } = options;

            if (!url || typeof url !== 'string') {
                return false;
            }

            // Check for unsafe schemes first
            if (this.isUnsafeScheme(url)) {
                return false;
            }

            // Handle fragment-only URLs
            if (url.startsWith('#')) {
                return allowFragments;
            }

            // Parse the URL
            try {
                const parsed = new URL(url, window.location.href);
                const scheme = parsed.protocol.slice(0, -1).toLowerCase(); // Remove trailing ':'

                // Check if it's a mailto link
                if (scheme === this.EMAIL_SCHEME) {
                    return allowMailto;
                }

                // Check if it's a safe scheme
                if (!this.SAFE_SCHEMES.includes(scheme)) {
                    console.warn(`Unsafe URL scheme: ${scheme}`);
                    return false;
                }

                // Validate domain if trusted domains are specified
                if (trustedDomains.length > 0 && parsed.hostname) {
                    const hostname = parsed.hostname.toLowerCase();
                    const isTrusted = trustedDomains.some(domain =>
                        hostname === domain.toLowerCase() ||
                        hostname.endsWith('.' + domain.toLowerCase())
                    );

                    if (!isTrusted) {
                        console.warn(`URL domain not in trusted list: ${parsed.hostname}`);
                        return false;
                    }
                }

                return true;
            } catch (e) {
                console.warn(`Failed to parse URL: ${e.message}`);
                return false;
            }
        },

        sanitizeUrl: function(url, defaultScheme = 'https') {
            if (!url) return null;

            // Check for unsafe schemes
            if (this.isUnsafeScheme(url)) {
                return null;
            }

            // Strip whitespace
            url = url.trim();

            // Add scheme if missing
            if (!url.match(/^[a-zA-Z][a-zA-Z\d+\-.]*:/)) {
                url = `${defaultScheme}://${url}`;
            }

            // Validate the final URL
            if (this.isSafeUrl(url, { requireScheme: true })) {
                return url;
            }

            return null;
        }
    };
    """