~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Offline latency probe for segmented FAISS at large vector counts.
Builds a collection of synthetic vectors through the production vector store
and a real encrypted SQLCipher database, without PDFs or an embedding model.
It then times per-document writes through ``VectorIndex.index_prepared`` and
searches through ``LibraryRAGService.search``, the paths whose cost grows
with collection size. The report contains numeric diagnostics only.
"""
from __future__ import annotations
import argparse
from contextlib import ExitStack
import hashlib
import json
import math
from pathlib import Path
import statistics
import sys
import tempfile
import time
import numpy as np
from local_deep_research.database.models.library import (
Collection,
Document,
DocumentCollection,
DocumentStatus,
SourceType,
)
from local_deep_research.research_library.services.library_rag_service import (
LibraryRAGService,
)
from local_deep_research.vector_stores.facade import ChunkInput
from local_deep_research.vector_stores.implementations import segmented_faiss
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from tests.performance._shared.resources import peak_rss_mib # noqa: E402
from tests.performance.rag_index_workload import ( # noqa: E402
EmbeddingManager,
require,
workload_session,
)
# Documents added per bulk-build transaction.
_BUILD_BATCH = 32
def _add_documents(session, source_id, collection_id, numbers):
for number in numbers:
document_id = f"bench-{number:07d}"
session.add(
Document(
id=document_id,
source_type_id=source_id,
title="Synthetic benchmark document",
document_hash=hashlib.sha256(document_id.encode()).hexdigest(),
text_content=document_id,
file_type="pdf",
file_size=len(document_id),
status=DocumentStatus.COMPLETED,
)
)
session.add(
DocumentCollection(
document_id=document_id, collection_id=collection_id
)
)
return [f"bench-{number:07d}" for number in numbers]
def _chunks(document_id, count):
return [
ChunkInput(
text=f"{document_id} chunk {position}",
metadata={"chunk_index": position},
)
for position in range(count)
]
def _seconds(values):
return {
"p50": round(statistics.median(values), 4),
"max": round(max(values), 4),
}
def run(args, root):
verify_known = getattr(args, "verify_known_query", False)
require(
not verify_known or args.dimension >= 8,
"Known-query checks require at least 8 dimensions",
)
rng = np.random.default_rng(7)
metrics = {
"target_vectors": args.vectors,
"dimension": args.dimension,
"index_type": args.index_type,
"chunks_per_document": args.chunks_per_document,
"orphan_scan_interval": segmented_faiss.ORPHAN_SCAN_INTERVAL,
}
started = time.monotonic()
with (
workload_session(root) as (session, key, _shared, _db_path),
ExitStack() as stack,
):
source = SourceType(
id="benchmark-source", name="document", display_name="Benchmark"
)
collection = Collection(id="benchmark", name="Vector scale benchmark")
session.add_all([source, collection])
session.commit()
source_id, collection_id = source.id, collection.id
rag = LibraryRAGService(
username="synthetic",
db_password=key,
embedding_model="synthetic-sha256",
embedding_provider="sentence_transformers",
embedding_manager=EmbeddingManager(args.dimension),
chunk_size=1000,
chunk_overlap=200,
index_type=args.index_type,
distance_metric="cosine",
normalize_vectors=True,
)
stack.callback(rag.close)
name = f"collection_{collection_id}"
index = rag._get_vector_index(collection_id, name)
# Bulk build through the same row and vector path as a write, but
# without the per-write orphan check: the build is setup, not the
# measured workload.
documents = math.ceil(args.vectors / args.chunks_per_document)
built = 0
for start in range(0, documents, _BUILD_BATCH):
numbers = range(start, min(documents, start + _BUILD_BATCH))
rows = []
for document_id in _add_documents(
session, source_id, collection_id, numbers
):
count = min(args.chunks_per_document, args.vectors - built)
rows.extend(
index._build_row(chunk, "document", document_id)
for chunk in _chunks(document_id, count)
)
built += count
session.add_all(rows)
session.flush()
index._apply(
session,
add_ids=[row.id for row in rows],
add_vectors=rng.standard_normal(
(len(rows), args.dimension), dtype=np.float32
),
)
session.commit()
del rows
if (start // _BUILD_BATCH) % 20 == 0:
print(json.dumps({"built_vectors": built}), flush=True)
metrics["build_seconds"] = round(time.monotonic() - started, 1)
metrics["initial_vectors"] = index.count()
metrics["segments"] = len(index._store._state())
require(metrics["initial_vectors"] == args.vectors, metrics)
# Per-document writes. The first one after the bulk build also runs
# the periodic full orphan scan, so it is reported separately.
write_seconds = []
numbers = range(documents, documents + args.writes)
_add_documents(session, source_id, collection_id, numbers)
session.commit()
known_document = f"bench-{documents:07d}"
known_query = "benchmark known-vector query"
metrics["writes_completed"] = 0
for number in numbers:
document_id = f"bench-{number:07d}"
vectors = rng.standard_normal(
(args.chunks_per_document, args.dimension), dtype=np.float32
)
if verify_known and number == documents:
vectors[0] = np.asarray(
index.embeddings.embed_query(known_query), dtype=np.float32
)
chunks = _chunks(document_id, args.chunks_per_document)
begun = time.monotonic()
stats = index.index_prepared(
source_type="document",
source_id=document_id,
chunks=chunks,
vectors=vectors,
)
write_seconds.append(time.monotonic() - begun)
require(stats.added == args.chunks_per_document, stats)
metrics["writes_completed"] += 1
metrics["final_vectors"] = index.count()
require(
metrics["final_vectors"]
== args.vectors + args.writes * args.chunks_per_document,
metrics,
)
entries = index._store._state()
metrics["largest_segment_vectors"] = max(
entry["count"] for entry in entries
)
require(
metrics["largest_segment_vectors"]
<= segmented_faiss.SEGMENT_VECTORS,
metrics,
)
metrics["first_write_seconds"] = round(write_seconds[0], 4)
metrics["write_seconds"] = _seconds(write_seconds[1:] or write_seconds)
# Searches through the service, as the research tools call them.
# The first search starts with no segment verified in this process.
segmented_faiss._VERIFIED.clear()
search_seconds = []
metrics["searches_completed"] = metrics["known_query_top1_matches"] = 0
for number in range(args.searches):
query = (
known_query
if verify_known and number == 0
else f"benchmark query {number}"
)
begun = time.monotonic()
hits = rag.search(query, collection_id, 10)
search_seconds.append(time.monotonic() - begun)
require(len(hits) == 10, len(hits))
require(
len({hit.chunk_id for hit in hits}) == 10,
"Duplicate search results",
)
require(
all(math.isfinite(hit.distance) for hit in hits),
"Non-finite search score",
)
if verify_known and number == 0:
require(
hits[0].source_id == known_document
and hits[0].text == f"{known_document} chunk 0",
"Known vector did not rank first",
)
metrics["known_query_top1_matches"] += 1
metrics["searches_completed"] += 1
metrics["cold_search_seconds"] = round(search_seconds[0], 4)
metrics["search_seconds"] = _seconds(
search_seconds[1:] or search_seconds
)
metrics["elapsed_seconds"] = round(time.monotonic() - started, 1)
metrics["peak_rss_mib"] = round(peak_rss_mib(), 2)
for limit, value, label in (
(args.max_write_seconds, metrics["write_seconds"]["p50"], "write"),
(args.max_search_seconds, metrics["search_seconds"]["p50"], "search"),
(args.max_rss_mib, metrics["peak_rss_mib"], "peak RSS"),
):
require(limit is None or value <= limit, (label, value, limit))
return metrics
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--vectors", type=int, default=20000)
parser.add_argument("--dimension", type=int, default=128)
parser.add_argument(
"--index-type", choices=("flat", "hnsw"), default="flat"
)
# 657 chunks per document matches the 1024-dimension library in #7628.
parser.add_argument("--chunks-per-document", type=int, default=657)
parser.add_argument("--writes", type=int, default=6)
parser.add_argument("--searches", type=int, default=8)
parser.add_argument("--max-write-seconds", type=float)
parser.add_argument("--max-search-seconds", type=float)
parser.add_argument("--max-rss-mib", type=float)
parser.add_argument("--verify-known-query", action="store_true")
parser.add_argument("--report", type=Path, required=True)
args = parser.parse_args()
if (
not 1000 <= args.vectors <= 10_000_000
or not 1 <= args.dimension <= 4096
or not 1 <= args.chunks_per_document <= 10_000
or args.writes < 1
or args.searches < 1
):
parser.error("Invalid benchmark size")
with tempfile.TemporaryDirectory(prefix="ldr-vector-scale-") as temporary:
metrics = run(args, Path(temporary))
args.report.parent.mkdir(parents=True, exist_ok=True)
args.report.write_text(
json.dumps(metrics, indent=2) + "\n", encoding="utf-8"
)
print(json.dumps(metrics), flush=True)
if __name__ == "__main__":
main()