Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

4.7 KB
Raw Blame
"""Filesystem lifecycle for vector-only segment directories (no FAISS import)."""

from pathlib import Path
import hashlib
import os
import re
import stat

GENERATED_SEGMENT_NAME = re.compile(
    r"[0-9a-f]{32}\.(?:faiss|ids)(?:\.[A-Za-z0-9_-]+\.tmp)?\Z"
)


def segment_directory(path: Path) -> Path:
    return Path(str(path) + ".segments")


def checkpoint_path(path: Path) -> Path:
    return Path(str(path) + ".rollback")


def recover_checkpoint(
    path: Path, checksum: str, algorithm: str = "sha256"
) -> bool:
    """Restore only bytes authenticated by the committed encrypted DB record.

    Never adopt the checkpoint's own checksum as a new baseline. A completed
    commit keeps the new root; an interrupted/rolled-back commit can recover
    the old one. Call outside an active vector write, under its index lock.
    """
    backup = checkpoint_path(path)
    if not backup.is_file() or backup.is_symlink() or path.is_symlink():
        return False
    if path.exists():
        with path.open("rb") as stream:
            if hashlib.file_digest(stream, algorithm).hexdigest() == checksum:
                return False
    with backup.open("rb") as stream:
        if hashlib.file_digest(stream, algorithm).hexdigest() != checksum:
            return False
    os.replace(backup, path)
    return True


def index_storage_size(path: Path) -> int | None:
    """Bytes held by an index root plus every regular file beside it.

    A segmented root is only a small manifest, so this adds every regular,
    non-symlink child of the segment directory (live segments, but also
    in-flight ``.tmp`` files, not-yet-retired segments and foreign files).
    Legacy single-file indexes have no segment directory. Readers hold no
    lock, so children (or the root) vanishing mid-scan are skipped rather
    than raised; returns None when the root itself is gone or unreadable.
    """
    try:
        size = path.stat().st_size
    except (OSError, ValueError):
        # Gone, or a corrupted path (ENOTDIR, ELOOP, ENAMETOOLONG, NUL).
        return None
    directory = segment_directory(path)
    if directory.is_symlink():
        return size
    try:
        with os.scandir(directory) as entries:
            for entry in entries:
                try:
                    info = entry.stat(follow_symlinks=False)
                except FileNotFoundError:
                    continue
                if stat.S_ISREG(info.st_mode):
                    size += info.st_size
    except OSError:
        # Missing, not a directory, or unreadable: report the root alone.
        pass
    return size


def delete_segment_files(path: Path) -> bool:
    """Retire generated files only; callers must validate the index root.

    Refuse foreign, symlinked or unexpected children and report unavailable
    components after attempting every removable generated sibling.
    """
    directory = segment_directory(path)
    if not directory.exists():
        return False
    if directory.is_symlink() or not directory.is_dir():
        raise ValueError(
            "Refusing a symlinked or invalid vector segment directory"
        )
    refused = False
    cleanup_error: OSError | None = None
    for child in directory.iterdir():
        try:
            # lstat: a symlink is never a regular file. A child that a
            # concurrent settle just retired is neither foreign nor an error.
            info = child.lstat()
            if not stat.S_ISREG(
                info.st_mode
            ) or not GENERATED_SEGMENT_NAME.fullmatch(child.name):
                refused = True
                continue
            child.unlink()
        except FileNotFoundError:
            continue
        except OSError as exc:
            if cleanup_error is None:
                cleanup_error = exc
    if cleanup_error is not None:
        raise cleanup_error
    if refused:
        raise ValueError("Refusing an unexpected vector segment file")
    # A file a concurrent writer added mid-walk makes this raise ENOTEMPTY,
    # which callers already report as incomplete cleanup.
    directory.rmdir()
    return True


def relocate_segments(source: Path, destination: Path) -> bool:
    """Move the companion directory with a relocated or quarantined root."""
    source_dir = segment_directory(source)
    if not source_dir.exists():
        return False
    if source_dir.is_symlink() or not source_dir.is_dir():
        raise ValueError(
            "Refusing a symlinked or invalid vector segment directory"
        )
    destination_dir = segment_directory(destination)
    if destination_dir.exists() or destination_dir.is_symlink():
        raise ValueError("Vector segment relocation destination already exists")
    source_dir.rename(destination_dir)
    return True