~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Filesystem lifecycle for vector-only segment directories (no FAISS import)."""
from pathlib import Path
import hashlib
import os
import re
import stat
GENERATED_SEGMENT_NAME = re.compile(
r"[0-9a-f]{32}\.(?:faiss|ids)(?:\.[A-Za-z0-9_-]+\.tmp)?\Z"
)
def segment_directory(path: Path) -> Path:
return Path(str(path) + ".segments")
def checkpoint_path(path: Path) -> Path:
return Path(str(path) + ".rollback")
def recover_checkpoint(
path: Path, checksum: str, algorithm: str = "sha256"
) -> bool:
"""Restore only bytes authenticated by the committed encrypted DB record.
Never adopt the checkpoint's own checksum as a new baseline. A completed
commit keeps the new root; an interrupted/rolled-back commit can recover
the old one. Call outside an active vector write, under its index lock.
"""
backup = checkpoint_path(path)
if not backup.is_file() or backup.is_symlink() or path.is_symlink():
return False
if path.exists():
with path.open("rb") as stream:
if hashlib.file_digest(stream, algorithm).hexdigest() == checksum:
return False
with backup.open("rb") as stream:
if hashlib.file_digest(stream, algorithm).hexdigest() != checksum:
return False
os.replace(backup, path)
return True
def index_storage_size(path: Path) -> int | None:
"""Bytes held by an index root plus every regular file beside it.
A segmented root is only a small manifest, so this adds every regular,
non-symlink child of the segment directory (live segments, but also
in-flight ``.tmp`` files, not-yet-retired segments and foreign files).
Legacy single-file indexes have no segment directory. Readers hold no
lock, so children (or the root) vanishing mid-scan are skipped rather
than raised; returns None when the root itself is gone or unreadable.
"""
try:
size = path.stat().st_size
except (OSError, ValueError):
# Gone, or a corrupted path (ENOTDIR, ELOOP, ENAMETOOLONG, NUL).
return None
directory = segment_directory(path)
if directory.is_symlink():
return size
try:
with os.scandir(directory) as entries:
for entry in entries:
try:
info = entry.stat(follow_symlinks=False)
except FileNotFoundError:
continue
if stat.S_ISREG(info.st_mode):
size += info.st_size
except OSError:
# Missing, not a directory, or unreadable: report the root alone.
pass
return size
def delete_segment_files(path: Path) -> bool:
"""Retire generated files only; callers must validate the index root.
Refuse foreign, symlinked or unexpected children and report unavailable
components after attempting every removable generated sibling.
"""
directory = segment_directory(path)
if not directory.exists():
return False
if directory.is_symlink() or not directory.is_dir():
raise ValueError(
"Refusing a symlinked or invalid vector segment directory"
)
refused = False
cleanup_error: OSError | None = None
for child in directory.iterdir():
try:
# lstat: a symlink is never a regular file. A child that a
# concurrent settle just retired is neither foreign nor an error.
info = child.lstat()
if not stat.S_ISREG(
info.st_mode
) or not GENERATED_SEGMENT_NAME.fullmatch(child.name):
refused = True
continue
child.unlink()
except FileNotFoundError:
continue
except OSError as exc:
if cleanup_error is None:
cleanup_error = exc
if cleanup_error is not None:
raise cleanup_error
if refused:
raise ValueError("Refusing an unexpected vector segment file")
# A file a concurrent writer added mid-walk makes this raise ENOTEMPTY,
# which callers already report as incomplete cleanup.
directory.rmdir()
return True
def relocate_segments(source: Path, destination: Path) -> bool:
"""Move the companion directory with a relocated or quarantined root."""
source_dir = segment_directory(source)
if not source_dir.exists():
return False
if source_dir.is_symlink() or not source_dir.is_dir():
raise ValueError(
"Refusing a symlinked or invalid vector segment directory"
)
destination_dir = segment_directory(destination)
if destination_dir.exists() or destination_dir.is_symlink():
raise ValueError("Vector segment relocation destination already exists")
source_dir.rename(destination_dir)
return True