~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Centralized URL validation utilities for security.
This module provides secure URL validation to prevent XSS attacks,
data exfiltration, and other URL-based security vulnerabilities.
"""
import re
from typing import Optional, List
from urllib.parse import urlparse, urljoin, unquote
from loguru import logger
from .log_sanitizer import redact_and_bound_for_log
class URLValidationError(ValueError):
"""Raised when URL construction or validation fails."""
pass
class URLValidator:
"""Centralized URL validation for security."""
# Unsafe URL schemes that could lead to XSS or data exfiltration
UNSAFE_SCHEMES = (
"javascript",
"data",
"vbscript",
"about",
"blob",
"file",
)
# Safe schemes for external links
SAFE_SCHEMES = ("http", "https", "ftp", "ftps")
# Email scheme
EMAIL_SCHEME = "mailto"
# Common academic/research domains that should be allowed
TRUSTED_ACADEMIC_DOMAINS = (
"arxiv.org",
"pubmed.ncbi.nlm.nih.gov",
"ncbi.nlm.nih.gov",
"biorxiv.org",
"medrxiv.org",
"doi.org",
"nature.com",
"science.org",
"sciencedirect.com",
"springer.com",
"wiley.com",
"plos.org",
"pnas.org",
"ieee.org",
"acm.org",
)
@staticmethod
def is_unsafe_scheme(url: str) -> bool:
"""
Check if a URL uses an unsafe scheme.
Args:
url: The URL to check
Returns:
True if the URL uses an unsafe scheme, False otherwise
"""
if not url:
return False
# Normalize the URL - trim whitespace and convert to lowercase
normalized_url = url.strip().lower()
# Check for unsafe schemes
for scheme in URLValidator.UNSAFE_SCHEMES:
if normalized_url.startswith(f"{scheme}:"):
logger.warning(
"Unsafe URL scheme detected: {} in URL: {}",
scheme,
# (#6938: redacted before it is cut, so the cut cannot
# drop the ``@`` after URL userinfo.)
redact_and_bound_for_log(url, 100),
)
return True
return False
@staticmethod
def is_safe_url(
url: str,
require_scheme: bool = True,
allow_fragments: bool = True,
allow_mailto: bool = False,
trusted_domains: Optional[List[str]] = None,
) -> bool:
"""
Validate if a URL is safe to use.
Args:
url: The URL to validate
require_scheme: Whether to require an explicit scheme
allow_fragments: Whether to allow fragment identifiers (#)
allow_mailto: Whether to allow mailto: links
trusted_domains: Optional list of trusted domains
Returns:
True if the URL is safe, False otherwise
"""
if not url or not isinstance(url, str):
return False
# Check for unsafe schemes first
if URLValidator.is_unsafe_scheme(url):
return False
# Handle fragment-only URLs
if url.startswith("#"):
return allow_fragments
# Parse the URL
try:
parsed = urlparse(url)
except Exception:
logger.warning(
"Failed to parse URL '{}'",
redact_and_bound_for_log(str(url), 100),
)
return False
# Check scheme
if not parsed.scheme:
if require_scheme:
return False
# If no scheme is required, assume http/https for URL parsing
parsed = urlparse(f"http://{url}") # DevSkim: ignore DS137138
scheme_lower = parsed.scheme.lower()
# Check if it's a mailto link
if scheme_lower == URLValidator.EMAIL_SCHEME:
return allow_mailto
# Check if it's a safe scheme
if scheme_lower not in URLValidator.SAFE_SCHEMES:
logger.warning(f"Unsafe URL scheme: {scheme_lower}")
return False
# Validate domain if trusted domains are specified
if trusted_domains and parsed.hostname:
hostname_lower = parsed.hostname.lower()
if not any(
hostname_lower == domain.lower()
or hostname_lower.endswith(f".{domain.lower()}")
for domain in trusted_domains
):
logger.warning(
f"URL domain not in trusted list: {parsed.hostname}"
)
return False
# Check for suspicious patterns in the URL
if URLValidator._has_suspicious_patterns(url):
return False
return True
@staticmethod
def _has_suspicious_patterns(url: str) -> bool:
"""
Check for suspicious patterns in URLs that might indicate attacks.
Args:
url: The URL to check
Returns:
True if suspicious patterns are found, False otherwise
"""
suspicious_patterns = [
# Double encoding
r"%25[0-9a-fA-F]{2}",
# Null bytes
r"%00",
# Unicode encoding bypass attempts
r"\\u[0-9a-fA-F]{4}",
# HTML entity encoding
r"&(#x?[0-9a-fA-F]+|[a-zA-Z]+);",
]
for pattern in suspicious_patterns:
if re.search(pattern, url, re.IGNORECASE):
logger.warning(f"Suspicious pattern found in URL: {pattern}")
return True
return False
@staticmethod
def sanitize_url(url: str, default_scheme: str = "https") -> Optional[str]:
"""
Sanitize a URL by adding a scheme if missing and validating it.
Args:
url: The URL to sanitize
default_scheme: The default scheme to add if missing
Returns:
Sanitized URL or None if the URL is unsafe
"""
if not url:
return None
# Check for unsafe schemes
if URLValidator.is_unsafe_scheme(url):
return None
# Strip whitespace
url = url.strip()
# Parse the URL
try:
parsed = urlparse(url)
# Add scheme if missing
if not parsed.scheme:
url = f"{default_scheme}://{url}"
parsed = urlparse(url)
# Validate the final URL
if URLValidator.is_safe_url(url, require_scheme=True):
return url
except Exception:
logger.warning(
"Failed to sanitize URL '{}'",
redact_and_bound_for_log(str(url), 100),
)
return None
@staticmethod
def is_academic_url(url: str) -> bool:
"""
Check if a URL is from a known academic/research domain.
Args:
url: The URL to check
Returns:
True if the URL is from an academic domain, False otherwise
"""
try:
parsed = urlparse(url)
if parsed.hostname:
hostname_lower = parsed.hostname.lower()
return any(
hostname_lower == domain
or hostname_lower.endswith(f".{domain}")
for domain in URLValidator.TRUSTED_ACADEMIC_DOMAINS
)
except Exception:
logger.debug(
"URL parsing may fail on malformed input; treat as non-academic",
exc_info=True,
)
return False
@staticmethod
def extract_doi(url: str) -> Optional[str]:
"""
Extract DOI from a URL if present.
Args:
url: The URL to extract DOI from
Returns:
The DOI if found, None otherwise
"""
# Common DOI patterns with explicit pattern identification
doi_patterns = [
(
r"10\.\d{4,}(?:\.\d+)*\/[-._;()\/:a-zA-Z0-9]+",
0,
), # Direct DOI, group 0
(r"doi\.org\/(10\.\d{4,}[^\s]*)", 1), # doi.org URL, group 1
]
for pattern, group_index in doi_patterns:
match = re.search(pattern, url, re.IGNORECASE)
if match:
return match.group(group_index)
return None
@staticmethod
def is_safe_http_link(url: object) -> bool:
"""Allow absolute HTTP(S) links without interpreting path/query data.
Literal percent signs, entity-like query values and trailing-dot hosts
are valid in browser links. Callback-specific suspicious-pattern checks
would reject them. Callers must still HTML-escape the URL in a quoted
attribute; this policy does not authorize server-side requests.
"""
if not isinstance(url, str) or not url:
return False
# Browsers and URL parsers can strip raw control characters. Reject
# those ambiguous inputs while preserving percent-encoded URL data.
if any(ord(char) < 0x20 or ord(char) == 0x7F for char in url):
return False
try:
parsed = urlparse(url)
# Accessing port also validates its numeric form and range.
port = parsed.port
return (
parsed.scheme in ("http", "https")
and bool(parsed.hostname)
and (port is None or 0 <= port <= 65535)
)
except ValueError:
return False
@staticmethod
def validate_http_url(url: str) -> bool:
"""
Validate that a callback URL is well-formed and safe for HTTP/HTTPS use.
This is stricter than is_safe_url() and specifically validates HTTP/HTTPS
URLs for use as application callbacks (e.g., in notifications, redirects).
It does NOT validate Apprise service URLs which use other protocols.
Args:
url: HTTP/HTTPS callback URL to validate
Returns:
True if valid
Raises:
URLValidationError: If URL is invalid
"""
if not url or not isinstance(url, str):
raise URLValidationError("URL must be a non-empty string")
try:
parsed = urlparse(url)
except URLValidationError:
raise
except Exception as e:
raise URLValidationError(f"Failed to validate URL: {e}") from e
# Must have a scheme
if not parsed.scheme:
raise URLValidationError("URL must have a scheme (http or https)")
# Must be http or https (callback URLs only)
if parsed.scheme not in ("http", "https"):
raise URLValidationError(
f"URL scheme must be http or https, got: {parsed.scheme}"
)
# Use the general security validator for additional safety
if not URLValidator.is_safe_url(url, require_scheme=True):
raise URLValidationError(f"URL failed security validation: {url}")
# Must have a netloc (hostname)
if not parsed.netloc:
raise URLValidationError("URL must have a hostname")
# Check for obvious hostname issues
if parsed.netloc.startswith(".") or parsed.netloc.endswith("."):
raise URLValidationError(f"Invalid hostname: {parsed.netloc}")
# Path should be valid if present
if parsed.path and not parsed.path.startswith("/"):
raise URLValidationError(
f"URL path must start with /: {parsed.path}"
)
return True
@staticmethod
def is_safe_redirect_url(target: str, host_url: str) -> bool:
"""
Validate that a redirect target is safe (same host).
Prevents open redirect attacks by ensuring the target URL
is either relative or points to the same host as the application.
Also prevents CRLF injection attacks by rejecting URLs containing
carriage return or line feed characters.
Uses the standard Flask pattern from:
https://github.com/fengsp/flask-snippets/blob/master/security/redirect_back.py
Security protections implemented:
- CRLF injection prevention
- Protocol-relative URL bypass (//evil.com)
- Triple-slash bypass (///evil.com)
- URL-encoded bypass attempts
- Backslash bypass (treated as forward slash by some browsers)
- Path traversal detection
Args:
target: The redirect URL to validate (can be relative or absolute)
host_url: The application's host URL (e.g., request.host_url)
Returns:
True if the URL is safe to redirect to, False otherwise
"""
if not target:
return False
# Prevent CRLF injection by rejecting URLs with CR or LF characters
if "\r" in target or "\n" in target:
logger.warning("CRLF injection attempt detected in redirect URL")
return False
# Normalize and decode URL to detect encoded bypass attempts
# This catches %2f%2f (encoded //) and similar tricks
decoded_target = unquote(target)
# Check for encoded CRLF injection (e.g. %0d%0a)
if "\r" in decoded_target or "\n" in decoded_target:
logger.warning(
"Encoded CRLF injection attempt detected in redirect URL"
)
return False
# Check for backslash (treated as forward slash by some browsers)
# Prevents \evil.com and \\evil.com bypasses
if "\\" in decoded_target:
logger.warning("Backslash detected in redirect URL")
return False
# Check for protocol-relative URLs before stripping slashes
# //evil.com would be treated as absolute URL by urljoin()
if decoded_target.startswith("//"):
logger.warning("Protocol-relative URL detected in redirect")
return False
# Block null bytes (truncation attacks in some URL processors)
if "\x00" in decoded_target:
logger.warning("Null byte detected in redirect URL")
return False
# Block path traversal patterns (/../, /.. at end, ../ at start)
if re.search(r"(^|/)\.\.(/|$|\?|#)", decoded_target):
logger.warning(
"Path traversal detected in redirect URL: {}",
# (#6938: the ``next`` target is a raw query param.)
redact_and_bound_for_log(target, 200),
)
return False
ref_url = urlparse(host_url)
test_url = urlparse(urljoin(host_url, target))
if test_url.path.startswith("//"):
logger.warning(
"Double-slash path in redirect URL: {}",
redact_and_bound_for_log(target, 200),
)
return False
return (
test_url.scheme in ("http", "https")
and ref_url.netloc == test_url.netloc
)
@staticmethod
def get_safe_redirect_path(target: str, host_url: str) -> Optional[str]:
"""
Validate a redirect target and return its path-only form.
Combines is_safe_redirect_url validation with defense-in-depth path
extraction. The target is resolved against host_url via urljoin, then
only the path, query, and fragment are returned. This ensures that
even a hypothetical validator bypass cannot cause an external redirect,
since path-only URLs always resolve to the same host.
Args:
target: The redirect URL to validate (can be relative or absolute)
host_url: The application's host URL (e.g., request.host_url)
Returns:
The safe path string (e.g., "/dashboard?tab=settings") if the
target is valid, or None if the target is unsafe.
"""
if not URLValidator.is_safe_redirect_url(target, host_url):
return None
resolved = urljoin(host_url, target)
parsed = urlparse(resolved)
safe_path = parsed.path or "/"
if parsed.query:
safe_path += "?" + parsed.query
if parsed.fragment:
safe_path += "#" + parsed.fragment
# Final assertion: the constructed path must be relative (no netloc).
# This is always true by construction but guards against future bugs.
if urlparse(safe_path).netloc:
return None
return safe_path
def get_javascript_url_validator() -> str:
"""
Get JavaScript code for URL validation that matches the Python implementation.
Returns:
JavaScript code as a string that can be embedded in web pages
"""
return r"""
// URL validation utilities matching Python URLValidator
const URLValidator = {
UNSAFE_SCHEMES: ['javascript', 'data', 'vbscript', 'about', 'blob', 'file'],
SAFE_SCHEMES: ['http', 'https', 'ftp', 'ftps'],
EMAIL_SCHEME: 'mailto',
isUnsafeScheme: function(url) {
if (!url) return false;
const normalizedUrl = url.trim().toLowerCase();
for (const scheme of this.UNSAFE_SCHEMES) {
if (normalizedUrl.startsWith(scheme + ':')) {
console.warn(`Unsafe URL scheme detected: ${scheme}`);
return true;
}
}
return false;
},
isSafeUrl: function(url, options = {}) {
const {
requireScheme = true,
allowFragments = true,
allowMailto = false,
trustedDomains = []
} = options;
if (!url || typeof url !== 'string') {
return false;
}
// Check for unsafe schemes first
if (this.isUnsafeScheme(url)) {
return false;
}
// Handle fragment-only URLs
if (url.startsWith('#')) {
return allowFragments;
}
// Parse the URL
try {
const parsed = new URL(url, window.location.href);
const scheme = parsed.protocol.slice(0, -1).toLowerCase(); // Remove trailing ':'
// Check if it's a mailto link
if (scheme === this.EMAIL_SCHEME) {
return allowMailto;
}
// Check if it's a safe scheme
if (!this.SAFE_SCHEMES.includes(scheme)) {
console.warn(`Unsafe URL scheme: ${scheme}`);
return false;
}
// Validate domain if trusted domains are specified
if (trustedDomains.length > 0 && parsed.hostname) {
const hostname = parsed.hostname.toLowerCase();
const isTrusted = trustedDomains.some(domain =>
hostname === domain.toLowerCase() ||
hostname.endsWith('.' + domain.toLowerCase())
);
if (!isTrusted) {
console.warn(`URL domain not in trusted list: ${parsed.hostname}`);
return false;
}
}
return true;
} catch (e) {
console.warn(`Failed to parse URL: ${e.message}`);
return false;
}
},
sanitizeUrl: function(url, defaultScheme = 'https') {
if (!url) return null;
// Check for unsafe schemes
if (this.isUnsafeScheme(url)) {
return null;
}
// Strip whitespace
url = url.trim();
// Add scheme if missing
if (!url.match(/^[a-zA-Z][a-zA-Z\d+\-.]*:/)) {
url = `${defaultScheme}://${url}`;
}
// Validate the final URL
if (this.isSafeUrl(url, { requireScheme: true })) {
return url;
}
return null;
}
};
"""