~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Offline release probe for synthetic PDF ingestion and bounded FAISS writes.
Run from the checkout with the project's test dependencies installed. The PDF
bytes stay in memory; document content and metadata use a real encrypted
SQLCipher database. Only deterministic embeddings and session routing are
substituted. Extraction, chunking, index IO, integrity checks, search and
reconciliation use production code. The report contains numeric diagnostics.
"""
from __future__ import annotations
import argparse
from contextlib import contextmanager, ExitStack
import hashlib
import json
import os
from pathlib import Path
import sys
import tempfile
import time
from unittest.mock import patch
import numpy as np
from langchain_core.embeddings import Embeddings
from loguru import logger
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlcipher3 import dbapi2
from local_deep_research.database.models import Base
from local_deep_research.database.models.library import (
Collection,
Document,
DocumentChunk,
DocumentCollection,
DocumentStatus,
SourceType,
)
from local_deep_research.document_loaders.bounded_pdf import load_bounded_pdf
from local_deep_research.research_library.services.library_rag_service import (
LibraryRAGService,
)
from local_deep_research.vector_stores.implementations import (
faiss_store,
segmented_faiss,
)
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from tests.fixtures.synthetic_pdf import synthetic_pdf # noqa: E402
from tests.performance._shared.resources import peak_rss_mib # noqa: E402
def require(condition, detail):
if not condition:
raise AssertionError(detail)
class SyntheticEmbeddings(Embeddings):
def __init__(self, dimension):
self.dimension = dimension
def embed_query(self, text):
seed = int.from_bytes(hashlib.sha256(text.encode()).digest()[:8], "big")
return np.random.default_rng(seed).random(self.dimension).tolist()
def embed_documents(self, texts):
return [self.embed_query(text) for text in texts]
class EmbeddingManager:
def __init__(self, dimension):
self.embeddings = SyntheticEmbeddings(dimension)
@contextmanager
def workload_session(root):
"""Yield a session on a real encrypted database that every caller shares.
Yields ``(session, key, shared_session, db_path)``. Production modules
that open their own user session receive this one instead.
"""
os.environ["LDR_DATA_DIR"] = str(root)
logger.disable("local_deep_research")
db_path = root / "documents.encrypted.db"
def connect():
connection = dbapi2.connect(str(db_path))
# Random raw test key: no production credentials or password KDF cost.
connection.execute(f'''PRAGMA key = "x'{key}'"''')
connection.execute("PRAGMA foreign_keys=ON")
connection.execute("PRAGMA cache_size=-4096")
return connection
key = os.urandom(32).hex()
engine = create_engine("sqlite://", creator=connect, module=dbapi2)
Base.metadata.create_all(engine)
session = sessionmaker(bind=engine)()
@contextmanager
def shared_session(*_args, **_kwargs):
yield session
with ExitStack() as stack:
stack.callback(engine.dispose)
stack.callback(session.close)
for module in (
"research_library.services.library_rag_service",
"vector_stores.facade",
"security.file_integrity.integrity_manager",
):
stack.enter_context(
patch(
f"local_deep_research.{module}.get_user_db_session",
shared_session,
)
)
stack.enter_context(
patch(
"local_deep_research.database.thread_local_session.get_current_thread_session",
return_value=session,
)
)
yield session, key, shared_session, db_path
def run(args, root):
metrics = {
"faiss_bytes_written": 0,
"metadata_bytes_written": 0,
"largest_segment_vectors": 0,
"document_count": args.documents,
"pages_per_document": args.pages,
"dimension": args.dimension,
"index_type": args.index_type,
}
original_write = faiss_store.write_index
original_atomic = segmented_faiss._atomic_write
def write_index(index, path):
metrics["largest_segment_vectors"] = max(
metrics["largest_segment_vectors"], int(index.ntotal)
)
require(
index.ntotal <= segmented_faiss.SEGMENT_VECTORS,
"Workload check failed: index.ntotal <= segmented_faiss.SEGMENT_VECTORS",
)
result = original_write(index, path)
metrics["faiss_bytes_written"] += Path(path).stat().st_size
return result
def atomic_write(path, data):
metrics["metadata_bytes_written"] += len(data)
return original_atomic(path, data)
started = time.monotonic()
with (
workload_session(root) as (session, key, shared_session, db_path),
ExitStack() as stack,
):
stack.enter_context(
patch.object(faiss_store, "write_index", write_index)
)
stack.enter_context(
patch.object(segmented_faiss, "_atomic_write", atomic_write)
)
source = SourceType(
id="synthetic-source",
name="document",
display_name="Synthetic PDFs",
)
collection = Collection(
id="synthetic-corpus", name="Synthetic PDF workload"
)
session.add_all([source, collection])
session.commit()
def service(chunk_size=1000):
return LibraryRAGService(
username="synthetic",
db_password=key,
embedding_model="synthetic-sha256",
embedding_provider="sentence_transformers",
embedding_manager=EmbeddingManager(args.dimension),
chunk_size=chunk_size,
chunk_overlap=200,
index_type=args.index_type,
distance_metric="cosine",
normalize_vectors=True,
)
rag = service()
stack.callback(rag.close)
document_ids = []
for number in range(args.documents):
pages = load_bounded_pdf(
synthetic_pdf(number, pages=args.pages), "synthetic.pdf"
)
require(
len(pages) == args.pages,
"Workload check failed: len(pages) == args.pages",
)
for page_number, page in enumerate(pages):
require(
not page.metadata.get("truncated"),
'Workload check failed: not page.metadata.get("truncated")',
)
require(
f"synthetic-document-{number:06d}-page-{page_number:04d}"
in page.page_content,
'Workload check failed: f"synthetic-document-{number:06d}-page-{page_number:04d}" in page.page_content',
)
text = "\n\n".join(page.page_content for page in pages)
document_id = f"synthetic-{number:06d}"
document_ids.append(document_id)
session.add(
Document(
id=document_id,
source_type_id=source.id,
title="Synthetic PDF",
document_hash=hashlib.sha256(text.encode()).hexdigest(),
text_content=text,
file_type="pdf",
file_size=len(text),
status=DocumentStatus.COMPLETED,
)
)
session.add(
DocumentCollection(
document_id=document_id, collection_id=collection.id
)
)
session.commit()
before = metrics["faiss_bytes_written"]
outcome = rag.index_document(document_id, collection.id)
require(outcome["status"] == "success", outcome)
# An append rewrites at most the bounded tail plus new segments.
written = metrics["faiss_bytes_written"] - before
bound = (
outcome["chunk_count"] + segmented_faiss.SEGMENT_VECTORS
) * (4 * args.dimension + 1024)
require(written <= bound, (written, bound))
session.expire_all()
if (number + 1) % 16 == 0:
print(json.dumps({"indexed_documents": number + 1}), flush=True)
name = f"collection_{collection.id}"
index = rag._get_vector_index(collection.id, name)
metrics["initial_vectors"] = index.count()
metrics["segments"] = len(index._store._state())
require(
metrics["initial_vectors"] > args.documents,
'Workload check failed: metrics["initial_vectors"] > args.documents',
)
for document_id in (
document_ids[0],
document_ids[len(document_ids) // 2],
document_ids[-1],
):
chunk = (
session.query(DocumentChunk)
.filter_by(
source_id=document_id, vector_index_key=index.index_key
)
.first()
)
require(
chunk is not None, "Workload check failed: chunk is not None"
)
hits = rag.search(chunk.chunk_text, collection.id, top_k=3)
require(
any(hit.chunk_id == chunk.id for hit in hits),
"Workload check failed: any(hit.chunk_id == chunk.id for hit in hits)",
)
# Roll back a replacement against the fully populated corpus, then
# reopen the store: its previous searchable generation must survive.
from local_deep_research.vector_stores.facade import ChunkInput
before_rollback = set(index.live_ids())
index.index(
source_type="document",
source_id=document_ids[0],
chunks=[ChunkInput(text="synthetic uncommitted replacement")],
session=session,
)
session.rollback()
index = rag._get_vector_index(collection.id, name)
require(
set(index.live_ids()) == before_rollback,
"Rollback changed corpus membership",
)
del before_rollback
# Full-configuration A -> B -> A must preserve the other index's rows.
alternate = service(1200)
stack.callback(alternate.close)
for configured in (alternate, rag):
result = configured.index_document(
document_ids[0], collection.id, force_reindex=True
)
require(result["status"] == "success", result)
for configured in (rag, alternate):
current = configured._get_vector_index(collection.id, name)
owned = {
row.id
for row in session.query(DocumentChunk.id)
.filter_by(vector_index_key=current.index_key)
.yield_per(500)
}
require(
set(current.live_ids()) == owned,
"Workload check failed: set(current.live_ids()) == owned",
)
# Bulk deletion purges both configurations, preserving unrelated docs.
removed = document_ids[: max(1, len(document_ids) // 4)]
require(
rag.purge_documents_vectors(removed, collection.id) > 0,
"Workload check failed: rag.purge_documents_vectors(removed, collection.id) > 0",
)
require(
session.query(DocumentChunk)
.filter(DocumentChunk.source_id.in_(removed))
.count()
== 0,
"Workload check failed: session.query(DocumentChunk).filter(DocumentChunk.source_id.in_(removed)).count() == 0",
)
for configured in (rag, alternate):
current = configured._get_vector_index(collection.id, name)
owned = {
row.id
for row in session.query(DocumentChunk.id)
.filter_by(vector_index_key=current.index_key)
.yield_per(500)
}
require(
set(current.live_ids()) == owned,
"Workload check failed: set(current.live_ids()) == owned",
)
reconciled = rag.reconcile_collection_index(collection.id)
require(
reconciled["orphan_vectors"] == 0,
'Workload check failed: reconciled["orphan_vectors"] == 0',
)
metrics["remaining_vectors"] = reconciled["live_vectors"]
# Finish the synthetic document deletion before rebuilding survivors.
session.query(DocumentCollection).filter(
DocumentCollection.document_id.in_(removed)
).delete(synchronize_session=False)
session.query(Document).filter(Document.id.in_(removed)).delete(
synchronize_session=False
)
session.commit()
document_ids = [value for value in document_ids if value not in removed]
index = rag._get_vector_index(collection.id, name)
# Rebuild through the production private-generation protocol. The
# old index stays searchable while every synthetic PDF is staged.
from local_deep_research.research_library.services.collection_rebuild import (
CollectionRebuild,
)
with CollectionRebuild(
rag,
collection.id,
[(value, "Synthetic PDF") for value in document_ids],
session_factory=shared_session,
) as rebuild:
for completed, document_id in enumerate(document_ids, 1):
prepared = rebuild.prepare(document_id)
require(prepared["status"] == "prepared", prepared)
rebuild.write(prepared["prepared"])
if completed % 16 == 0:
print(
json.dumps({"staged_documents": completed}), flush=True
)
require(
index.count() == metrics["remaining_vectors"],
"Staging changed the published corpus",
)
rebuild.publish()
index = rag._get_vector_index(collection.id, name)
metrics["rebuilt_vectors"] = index.count()
require(
metrics["rebuilt_vectors"] == metrics["remaining_vectors"],
"Rebuild lost vectors",
)
metrics["elapsed_seconds"] = round(time.monotonic() - started, 3)
metrics["peak_rss_mib"] = round(peak_rss_mib(), 2)
require(metrics["peak_rss_mib"] <= args.max_rss_mib, metrics)
# No generated PDF or plaintext chunk cache may have reached disk.
for path in root.rglob("*"):
if path.is_file():
with path.open("rb") as stream:
overlap = b""
for block in iter(lambda: stream.read(1024 * 1024), b""):
require(
b"synthetic-document-" not in overlap + block, path
)
overlap = block[-32:]
with db_path.open("rb") as stream:
require(
stream.read(16) != b"SQLite format 3\x00",
"Document database is plaintext",
)
return metrics
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--documents", type=int, default=4)
parser.add_argument("--pages", type=int, default=2)
parser.add_argument("--dimension", type=int, default=128)
parser.add_argument(
"--index-type", choices=("flat", "hnsw"), default="hnsw"
)
parser.add_argument("--max-rss-mib", type=int, default=768)
parser.add_argument("--report", type=Path, required=True)
args = parser.parse_args()
if not 2 <= args.documents <= 10000 or not 1 <= args.dimension <= 4096:
parser.error("Invalid workload size")
with tempfile.TemporaryDirectory(prefix="ldr-rag-workload-") as temporary:
metrics = run(args, Path(temporary))
args.report.parent.mkdir(parents=True, exist_ok=True)
args.report.write_text(
json.dumps(metrics, indent=2) + "\n", encoding="utf-8"
)
print(json.dumps(metrics), flush=True)
if __name__ == "__main__":
main()