~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
URL building utilities for security and application use.
Provides centralized URL construction logic that can be reused
throughout the application for consistent URL handling.
"""
import re
from typing import Optional, Union
from urllib.parse import urlparse
from loguru import logger
from .ssrf_validator import redact_url_for_log
class URLBuilderError(Exception):
"""Raised when URL construction fails."""
pass
def normalize_bind_address(host: str) -> str:
"""
Convert bind addresses to URL-friendly hostnames.
Args:
host: Host address from settings (may include bind addresses)
Returns:
URL-friendly hostname
"""
# Convert bind-all addresses to localhost for URLs
# Replace a bind-all address with localhost for a client URL.
if host in ("0.0.0.0", "::"): # nosec B104
return "localhost"
return host
def build_base_url_from_settings(
external_url: Optional[str] = None,
host: Optional[str] = None,
port: Optional[Union[str, int]] = None,
fallback_base: str = "http://localhost:5000",
) -> str:
"""
Build a base URL from application settings with intelligent fallbacks.
This function handles the common pattern of building application URLs
from various configuration sources with proper normalization.
Args:
external_url: Pre-configured external URL (highest priority)
host: Hostname/IP address (used if external_url not provided)
port: Port number (used with host if external_url not provided)
fallback_base: Final fallback URL if nothing else is available
Returns:
Complete base URL (e.g., "https://myapp.com" or "http://localhost:5000")
Raises:
URLBuilderError: If URL construction fails
"""
try:
# Try external URL first (highest priority)
if external_url and external_url.strip():
base_url = external_url.strip().rstrip("/")
logger.debug(
f"Using configured external URL: {redact_url_for_log(base_url)}"
)
return base_url
# Try to construct from host and port
if host and port:
normalized_host = normalize_bind_address(host)
# Use HTTP for host/port combinations (typically internal server addresses)
# For external URLs, users should configure external_url setting instead
base_url = f"http://{normalized_host}:{int(port)}" # DevSkim: ignore DS137138
logger.debug(
f"Constructed URL from host/port: {redact_url_for_log(base_url)}"
)
return base_url
# Final fallback
base_url = fallback_base.rstrip("/")
logger.debug(f"Using fallback URL: {redact_url_for_log(base_url)}")
return base_url
except Exception as e:
raise URLBuilderError(f"Failed to build base URL: {e}")
def build_full_url(
base_url: str,
path: str,
validate: bool = True,
allowed_schemes: Optional[list] = None,
) -> str:
"""
Build a complete URL from base URL and path.
Args:
base_url: Base URL (e.g., "https://myapp.com")
path: Path to append (e.g., "/research/123")
validate: Whether to validate the resulting URL
allowed_schemes: List of allowed URL schemes (default: ["http", "https"])
Returns:
Complete URL (e.g., "https://myapp.com/research/123")
Raises:
URLBuilderError: If URL construction or validation fails
"""
try:
# Ensure path starts with /
if not path.startswith("/"):
path = f"/{path}"
# Ensure base URL doesn't end with /
base_url = base_url.rstrip("/")
# Construct full URL
full_url = f"{base_url}{path}"
if validate:
validate_constructed_url(full_url, allowed_schemes)
return full_url
except Exception as e:
raise URLBuilderError(f"Failed to build full URL: {e}")
def validate_constructed_url(
url: str, allowed_schemes: Optional[list] = None
) -> bool:
"""
Validate a constructed URL.
Args:
url: URL to validate
allowed_schemes: List of allowed schemes (default: ["http", "https"])
Returns:
True if valid
Raises:
URLBuilderError: If URL is invalid
"""
if not url or not isinstance(url, str):
raise URLBuilderError("URL must be a non-empty string")
try:
parsed = urlparse(url)
except Exception as e:
raise URLBuilderError(f"Failed to parse URL: {e}")
# Check scheme
if not parsed.scheme:
raise URLBuilderError("URL must have a scheme")
if allowed_schemes and parsed.scheme not in allowed_schemes:
raise URLBuilderError(
f"URL scheme '{parsed.scheme}' not in allowed schemes: {allowed_schemes}"
)
# Check hostname
if not parsed.netloc:
raise URLBuilderError("URL must have a hostname")
return True
def mask_sensitive_url(url: str) -> str:
"""
Mask sensitive parts of a URL for secure logging.
This function masks passwords, webhook tokens, and other sensitive
information in URLs to prevent accidental exposure in logs.
Args:
url: URL to mask
Returns:
URL with sensitive parts replaced with ***
"""
# A non-string never reaches the masking logic usefully: urlparse(None)
# and urlparse([]) do NOT raise, they return empty byte-ish parts, so the
# function used to emit nonsense like "b''://b''b''" -- unmasked and
# meaningless. Non-strings also broke the except-branch, which does
# url.split(':') and raises AttributeError from inside the very helper
# that exists to make logging an unexpected value safe. There is no
# scheme worth preserving here, so mask the whole thing.
if not isinstance(url, str):
return "***"
try:
parsed = urlparse(url)
# Mask password if present
if parsed.password:
netloc = parsed.netloc.replace(parsed.password, "***")
else:
netloc = parsed.netloc
# Mask path tokens (common in webhooks)
path = parsed.path
if path:
# Replace long alphanumeric tokens with ***
path = re.sub(
r"/[a-zA-Z0-9_-]{20,}",
"/***",
path,
)
# Reconstruct URL
masked = f"{parsed.scheme}://{netloc}{path}"
if parsed.query:
masked += "?***"
return masked
except Exception:
# If parsing fails, return a generic mask. ``url`` is a str by the
# time it reaches here -- non-strings returned "***" above -- so the
# split cannot itself raise. That matters: this handler exists to
# make logging an unexpected value safe, so it must never be the
# thing that fails. A value with no scheme masks entirely rather
# than emitting a bare "://***".
scheme = url.split(":", 1)[0]
return f"{scheme}://***" if scheme else "***"