Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
import importlib.util
import sys
from pathlib import Path
import numpy as np
import pytest
MODULE_PATH = Path(__file__).resolve().parents[2] / "benchmark" / "cuvs" / "run_index_benchmark.py"
SPEC = importlib.util.spec_from_file_location("run_cuvs_index_benchmark", MODULE_PATH)
assert SPEC is not None and SPEC.loader is not None
benchmark = importlib.util.module_from_spec(SPEC)
sys.modules[SPEC.name] = benchmark
SPEC.loader.exec_module(benchmark)
def test_percentile_interpolates_and_recall_ignores_order():
assert benchmark.percentile([1.0, 2.0, 3.0, 4.0], 0.5) == pytest.approx(2.5)
actual = np.asarray([[2, 1, 8], [4, 5, 6]])
expected = np.asarray([[1, 2, 3], [4, 7, 8]])
assert benchmark.recall_at_k(actual, expected, 3) == pytest.approx(0.5)
def test_prepare_dataset_is_normalized_deterministic_and_reusable(tmp_path):
arguments = {
"vector_count": 32,
"dimension": 8,
"query_count": 5,
"metric": "cosine",
"seed": 7,
"generation_chunk_size": 11,
"force": False,
}
first = benchmark.prepare_dataset(tmp_path, **arguments)
dataset = np.load(first.dataset, mmap_mode="r", allow_pickle=False)
queries = np.load(first.queries, mmap_mode="r", allow_pickle=False)
assert first.reused is False
assert dataset.shape == (32, 8)
assert queries.shape == (5, 8)
np.testing.assert_allclose(np.linalg.norm(dataset, axis=1), 1.0, atol=1e-5)
np.testing.assert_allclose(np.linalg.norm(queries, axis=1), 1.0, atol=1e-5)
second = benchmark.prepare_dataset(tmp_path, **arguments)
assert second.reused is True
assert second.generated_seconds == 0.0
np.testing.assert_array_equal(dataset, np.load(second.dataset, allow_pickle=False))
def test_prepare_ann_benchmarks_dataset_converts_normalizes_and_reuses(tmp_path):
h5py = pytest.importorskip("h5py")
source = tmp_path / "tiny-angular.hdf5"
with h5py.File(source, "w") as hdf5:
hdf5.attrs["distance"] = "angular"
hdf5.create_dataset(
"train",
data=np.asarray([[3.0, 4.0], [1.0, 0.0], [0.0, 2.0]], dtype=np.float32),
)
hdf5.create_dataset(
"test",
data=np.asarray([[6.0, 8.0], [2.0, 0.0]], dtype=np.float32),
)
hdf5.create_dataset("neighbors", data=np.asarray([[0, 1], [1, 0]], dtype=np.int32))
arguments = {
"source": source,
"metric": "cosine",
"vector_limit": None,
"query_limit": 1,
"generation_chunk_size": 2,
"force": False,
}
first = benchmark.prepare_ann_benchmarks_dataset(tmp_path / "data", **arguments)
dataset = np.load(first.dataset, mmap_mode="r", allow_pickle=False)
queries = np.load(first.queries, mmap_mode="r", allow_pickle=False)
ground_truth = np.load(first.ground_truth, mmap_mode="r", allow_pickle=False)
assert first.reused is False
assert dataset.shape == (3, 2)
assert queries.shape == (1, 2)
np.testing.assert_allclose(np.linalg.norm(dataset, axis=1), 1.0, atol=1e-6)
np.testing.assert_allclose(np.linalg.norm(queries, axis=1), 1.0, atol=1e-6)
np.testing.assert_array_equal(ground_truth, np.asarray([[0, 1]]))
metadata = first.metadata.read_text()
assert str(source) not in metadata
assert '"source_sha256"' in metadata
second = benchmark.prepare_ann_benchmarks_dataset(tmp_path / "data", **arguments)
assert second.reused is True
assert second.generated_seconds == 0.0
def test_cagra_itopk_sweep_preserves_shared_search_params():
parser = benchmark.build_parser()
args = parser.parse_args(
[
"--cagra-search-params",
'{"search_width":2}',
"--cagra-itopk-sizes",
"32,64,128",
]
)
assert benchmark.cagra_search_variants(args) == [
{"search_width": 2, "itopk_size": 32},
{"search_width": 2, "itopk_size": 64},
{"search_width": 2, "itopk_size": 128},
]
def test_validate_args_accepts_float16_backend_variants():
parser = benchmark.build_parser()
args = parser.parse_args(
["--backends", "cuvs_brute_force,cuvs_brute_force_fp16,cuvs_cagra_fp16"]
)
assert benchmark.validate_args(parser, args) == [
"cuvs_brute_force",
"cuvs_brute_force_fp16",
"cuvs_cagra_fp16",
]
@pytest.mark.parametrize(
"backends",
[
["cuvs_brute_force_fp16"],
["cuvs_cagra"],
["cuvs_cagra_fp16"],
["cuvs_brute_force_fp16", "cuvs_cagra"],
],
)
def test_lossy_or_approximate_backend_requires_exact_reference(backends):
parser = benchmark.build_parser()
with pytest.raises(SystemExit) as exc_info:
benchmark.validate_reference_requirement(
parser,
backends,
has_supplied_ground_truth=False,
)
assert exc_info.value.code == 2
@pytest.mark.parametrize(
"backends",
[
["native"],
["cuvs_brute_force"],
["native", "cuvs_brute_force"],
["native", "cuvs_cagra"],
["cuvs_brute_force", "cuvs_brute_force_fp16"],
],
)
def test_reference_validation_accepts_exact_backend_combinations(backends):
parser = benchmark.build_parser()
benchmark.validate_reference_requirement(
parser,
backends,
has_supplied_ground_truth=False,
)
@pytest.mark.parametrize(
"backend",
["cuvs_brute_force_fp16", "cuvs_cagra", "cuvs_cagra_fp16"],
)
def test_reference_validation_accepts_supplied_ground_truth(backend):
parser = benchmark.build_parser()
benchmark.validate_reference_requirement(
parser,
[backend],
has_supplied_ground_truth=True,
)
def test_print_summary_does_not_invent_missing_recall(capsys):
benchmark.print_summary(
[
{
"backend": "cuvs_brute_force_fp16",
"build_seconds": 1.0,
"first_search_per_query_ms": 2.0,
"search": {
"per_query_latency_ms": {"p50": 3.0, "p95": 4.0},
"qps": 5.0,
},
}
]
)
output = capsys.readouterr().out
assert "N/A" in output
assert "1.0000" not in output
def test_cagra_itopk_and_search_width_sweeps_form_cartesian_product():
parser = benchmark.build_parser()
args = parser.parse_args(
[
"--cagra-itopk-sizes",
"64,128",
"--cagra-search-widths",
"1,4",
]
)
assert benchmark.cagra_search_variants(args) == [
{"itopk_size": 64, "search_width": 1},
{"itopk_size": 128, "search_width": 1},
{"itopk_size": 64, "search_width": 4},
{"itopk_size": 128, "search_width": 4},
]
def test_run_search_records_batches_and_preserves_query_order():
class FakeBackend:
def search(self, queries, k):
labels = np.repeat(queries[:, :1].astype(np.int64), k, axis=1)
return labels, np.zeros_like(labels, dtype=np.float32)
queries = np.arange(5, dtype=np.float32).reshape(5, 1)
neighbors, summary = benchmark.run_search(
FakeBackend(),
queries,
k=2,
batch_size=2,
warmup_batches=1,
repetitions=2,
)
np.testing.assert_array_equal(neighbors[:, 0], np.arange(5))
assert summary["unique_query_count"] == 5
assert summary["timed_query_count"] == 10
assert summary["batch_count"] == 6
assert len(summary["raw_batch_latency_ms"]) == 6
assert summary["qps"] > 0