Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

13.3 KB
Raw Blame
"""
Centralized path configuration for Local Deep Research.
Handles database location using platformdirs for proper user data storage.
"""

import errno
import hashlib
import os
import stat
from pathlib import Path

import platformdirs
from loguru import logger


def get_data_directory() -> Path:
    """
    Get the appropriate data directory for storing application data.
    Uses platformdirs to get platform-specific user data directory.

    Environment variable:
        LDR_DATA_DIR: Override the default data directory location.
                     All subdirectories (research_outputs, cache, logs, database)
                     will be created under this directory.

    Returns:
        Path to data directory
    """
    # Check for explicit override via environment variable
    custom_path = os.getenv("LDR_DATA_DIR")
    if custom_path:
        if not Path(custom_path).is_absolute():
            raise ValueError("LDR_DATA_DIR must be an absolute path")
        data_dir = Path(custom_path).resolve()
        # Reject only control characters that are never valid in a path and
        # could break a SQL/ATTACH statement (null byte, newline, carriage
        # return). Quotes are intentionally NOT rejected here: legitimate POSIX
        # home directories contain apostrophes (e.g. /home/O'Brien/ldr), and the
        # actual ATTACH-DATABASE sink (backup_service) already validates the
        # full interpolated path against a stricter denylist before use.
        # The offending path is not echoed, mirroring the redaction below.
        path_str = str(data_dir)
        if any(c in path_str for c in ("\0", "\n", "\r")):
            raise ValueError(
                "LDR_DATA_DIR contains unsafe control characters "
                "(null byte, newline, or carriage return)"
            )
        logger.debug(
            f"Using custom data directory from LDR_DATA_DIR: {data_dir}"
        )
        return data_dir

    # Use platformdirs for platform-specific user data directory
    # Windows: C:\Users\Username\AppData\Local\local-deep-research
    # macOS: ~/Library/Application Support/local-deep-research
    # Linux: ~/.local/share/local-deep-research
    data_dir = Path(platformdirs.user_data_dir("local-deep-research"))
    # Log only the directory pattern, not the full path which may contain username
    logger.debug(
        f"Using platformdirs data directory pattern: .../{data_dir.name}"
    )

    return data_dir


def get_custom_data_directory() -> Path | None:
    """
    Get the data directory that ``LDR_DATA_DIR`` selects, if it is set.

    ``None`` means ``LDR_DATA_DIR`` is unset or empty, so
    :func:`get_data_directory` uses the platform default. Otherwise the result
    is exactly what :func:`get_data_directory` returns, including its
    validation errors. ``LDR_DATA_DIR`` is the only environment variable that
    moves the data directory, so this is what a status report should show
    when it says the location is custom.

    Returns:
        Path selected by ``LDR_DATA_DIR``, or ``None`` for the platform default
    """
    if not os.getenv("LDR_DATA_DIR"):
        return None
    return get_data_directory()


def _ensure_dir(
    subdir: str, label: str | None = None, mode: int | None = None
) -> Path:
    """Create (if needed) and return ``<data_dir>/<subdir>``.

    Args:
        subdir: Path segment below the data directory. May be multi-segment
            (e.g. ``"encrypted_databases/backups"``).
        label: If provided, emit a debug log ``"Using {label} directory:
            {path}"``. ``None`` suppresses logging.
        mode: Optional creation mode for a new directory.

    Returns:
        The ensured directory path.
    """
    path = get_data_directory() / subdir
    if mode is None:
        path.mkdir(  # Safe: bootstrap data subdirectory
            parents=True, exist_ok=True
        )
    else:
        path.mkdir(  # Safe: bootstrap data subdirectory
            parents=True, exist_ok=True, mode=mode
        )
    if label is not None:
        logger.debug(f"Using {label} directory: {path}")
    return path


#: ``DirectoryNotPrivateError.reason`` values.
OWNED_BY_ANOTHER_ACCOUNT = "owned_by_another_account"
MODE_NOT_KEPT = "mode_not_kept"

#: What an operator can do when report file backups are refused.
REPORT_BACKUP_REMEDY = (
    "Keep the data directory (LDR_DATA_DIR) on a Linux filesystem path or a "
    "Docker named volume owned by the account that runs Local Deep Research, "
    "or turn off report.enable_file_backup"
)


class DirectoryNotPrivateError(PermissionError):
    """A directory cannot be restricted to the current OS account.

    ``reason`` is ``OWNED_BY_ANOTHER_ACCOUNT`` or ``MODE_NOT_KEPT``.
    """

    def __init__(self, message: str, reason: str):
        super().__init__(message)
        self.reason = reason


def restrict_directory_to_owner(path: Path) -> None:
    """Set ``path`` to mode 0700 on POSIX and verify the result.

    Other systems have no POSIX mode bits, so this does nothing there; the
    directory's access-control list decides who can read it.

    The result is checked because some mounts accept chmod without keeping
    it, or show another owner: Docker Desktop bind mounts of Windows
    folders, WSL drives such as /mnt/c, CIFS/SMB shares and exFAT.

    Raises:
        DirectoryNotPrivateError: another OS account owns the directory, or
            its mode is not exactly 0700 after chmod.
        OSError: chmod or stat failed for another reason.
    """
    if os.name != "posix":
        return
    try:
        path.chmod(0o700)
    except PermissionError:
        # Only the owner may chmod. Report a foreign owner as such below;
        # any other cause propagates unchanged.
        if path.stat().st_uid == os.geteuid():
            raise
    info = path.stat()
    if info.st_uid != os.geteuid():
        raise DirectoryNotPrivateError(
            f"{path} is not private: it is owned by uid {info.st_uid}, "
            f"but this process runs as uid {os.geteuid()}",
            OWNED_BY_ANOTHER_ACCOUNT,
        )
    mode = stat.S_IMODE(info.st_mode)
    if mode != 0o700:
        raise DirectoryNotPrivateError(
            f"{path} is not private: its filesystem kept mode {mode:04o} "
            "after chmod 0700",
            MODE_NOT_KEPT,
        )


def describe_private_directory_refusal(error: OSError) -> str:
    """Say why ``restrict_directory_to_owner`` refused a directory.

    Fixed text for log lines: it never echoes the error message, only its
    category and, for other OS errors, the symbolic errno.
    """
    reason = getattr(error, "reason", None)
    if reason == OWNED_BY_ANOTHER_ACCOUNT:
        return "another OS account owns it"
    if reason == MODE_NOT_KEPT:
        return "its filesystem did not keep mode 0700"
    code = errno.errorcode.get(getattr(error, "errno", None) or 0)
    return "it could not be created or set to mode 0700" + (
        f" ({code})" if code else ""
    )


def get_research_outputs_directory() -> Path:
    """Get the directory for research outputs and optional report backups.

    A missing directory is created with mode 0700 (subject to the umask).
    An existing directory is returned unchanged: the web application
    restricts it at startup (``restrict_research_outputs_directory``), and
    ``FileReportStorage`` restricts and verifies it before writing.

    Returns:
        Path to research outputs directory
    """
    return _ensure_dir("research_outputs", label="research outputs", mode=0o700)


def restrict_research_outputs_directory() -> bool:
    """Restrict the research outputs directory to this OS account.

    The web application calls this once at startup, so a directory left at
    0755 by an older release becomes 0700 even when no user has report file
    backups enabled. An ``OSError`` is logged with its remedy, not raised:
    ``FileReportStorage`` then refuses to write report backups there, and
    database report storage is unaffected.

    Returns:
        False when the directory could not be made private, otherwise True
        (always True on non-POSIX systems, where nothing is changed).
    """
    try:
        restrict_directory_to_owner(get_research_outputs_directory())
    except OSError as exc:
        cause = describe_private_directory_refusal(exc)
        logger.warning(
            f"Could not make the research outputs directory private: {cause}. "
            "Report file backups will be refused until this is fixed; "
            f"reports are still saved in the database. {REPORT_BACKUP_REMEDY}."
        )
        return False
    return True


def get_journal_data_directory() -> Path:
    """Get the directory for downloaded journal quality data files.

    Contains openalex_sources.json.gz, doaj_journals.json, and the
    compiled journal_reference.db. Fetched on first use from
    OpenAlex and DOAJ APIs.

    Returns:
        Path to journal data directory
    """
    return _ensure_dir("journal_data")


def get_cache_directory() -> Path:
    """
    Get the directory for storing cache files (search cache, etc.).

    Returns:
        Path to cache directory
    """
    return _ensure_dir("cache", label="cache")


def get_logs_directory() -> Path:
    """
    Get the directory for storing log files.

    Returns:
        Path to logs directory
    """
    return _ensure_dir("logs", label="logs")


def get_encrypted_database_path() -> Path:
    """Get the path to the encrypted databases directory.

    Returns:
        Path to the encrypted databases directory
    """
    return _ensure_dir("encrypted_databases")


def get_user_database_filename(username: str) -> str:
    """Get the database filename for a specific user.

    Args:
        username: The username to generate a filename for

    Returns:
        The database filename (not full path) for the user

    Raises:
        ValueError: If username is empty/None. Registration already enforces
            a non-empty username, but this is the actual hash chokepoint:
            sha256("") is a well-known constant, so silently accepting an
            empty username would let two different "no username" call sites
            collide into one shared, deterministic database filename instead
            of failing loudly. Defense-in-depth -- see #5481.
    """
    if not username:
        raise ValueError(
            "Cannot compute a user database filename for an empty username"
        )
    # Use username hash to avoid filesystem issues with special characters
    username_hash = hashlib.sha256(username.encode()).hexdigest()[:16]
    return f"ldr_user_{username_hash}.db"


def get_library_directory() -> Path:
    """
    Get the directory for storing library files (documents, PDFs, etc.).

    Returns:
        Path to library directory
    """
    return _ensure_dir("library", label="library")


def get_config_directory() -> Path:
    """
    Get the directory for storing configuration files.

    Returns:
        Path to config directory
    """
    return _ensure_dir("config", label="config")


def get_models_directory() -> Path:
    """
    Get the directory for storing downloaded models.

    Returns:
        Path to models directory
    """
    return _ensure_dir("models", label="models")


def get_sentence_transformers_home() -> str | None:
    """
    Get the sentence-transformers ecosystem cache override, if set.

    Environment variable:
        SENTENCE_TRANSFORMERS_HOME: Third-party override (honored by the
                     sentence-transformers library itself) for where model
                     snapshots are stored. Read at call time so the live
                     environment always wins, mirroring how the library
                     resolves its own cache root. The raw value is returned
                     (no Path coercion, empty string stays empty) so callers
                     can preserve the library's precedence exactly.

    Returns:
        The override path as a string, or None when unset (callers fall
        back to the HuggingFace Hub default cache).
    """
    return os.getenv("SENTENCE_TRANSFORMERS_HOME")


def get_backup_directory() -> Path:
    """Get the base backup directory for all users."""
    return _ensure_dir("encrypted_databases/backups")


def get_user_backup_directory(username: str) -> Path:
    """Get backup directory for a specific user.

    Raises:
        ValueError: If username is empty/None. Same fail-closed guard as
            get_user_database_filename: sha256("") is a well-known constant,
            so silently accepting an empty username would collapse every
            "no username" call site into one shared, deterministic backup
            directory. Defense-in-depth -- see #5481.
    """
    if not username:
        raise ValueError(
            "Cannot compute a user backup directory for an empty username"
        )
    from local_deep_research.security.directory_creation import create_directory

    username_hash = hashlib.sha256(username.encode()).hexdigest()[:16]
    backup_directory = get_backup_directory()
    user_backup_dir = create_directory(
        backup_directory / username_hash,
        context="user backup directory",
        root=backup_directory,
        mode=0o700,
    )
    # Enforce 0o700 regardless of umask (mkdir mode is umask-masked)
    user_backup_dir.chmod(0o700)
    return user_backup_dir


# Convenience functions for backward compatibility
def get_data_dir() -> str:
    """Get data directory as string for backward compatibility."""
    return str(get_data_directory())