Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
import importlib.util
import json
import sys
from pathlib import Path
import pytest
MODULE_PATH = (
Path(__file__).resolve().parents[2] / "benchmark" / "cuvs" / "summarize_collection_runs.py"
)
SPEC = importlib.util.spec_from_file_location("summarize_cuvs_collection_runs", MODULE_PATH)
assert SPEC is not None and SPEC.loader is not None
summary_module = importlib.util.module_from_spec(SPEC)
sys.modules[SPEC.name] = summary_module
SPEC.loader.exec_module(summary_module)
def result_document(*, qps, p50, timestamp="run-1", dimension=8):
return {
"format_version": 1,
"runtime": {
"timestamp": timestamp,
"git_revision": "abc123",
"git_dirty": False,
"gpu": {"name": "test-gpu"},
"cpu_model": "test-cpu",
"cuvs": "1.0",
"cupy": "2.0",
},
"dataset": {
"kind": "random",
"vector_count": 100,
"dimension": dimension,
"query_count": 10,
"metric": "cosine",
"seed": 42,
},
"parameters": {"k": 10, "mutation_sizes": [1]},
"results": [
{
"backend": "cuvs_brute_force",
"ingest": {"total_seconds": 2.0, "records_per_second": 50.0},
"rss_ingest_delta_bytes": 1000,
"gpu_search_delta_bytes": 2000,
"searches": [
{
"name": "unfiltered",
"filter": None,
"distribution": "none",
"target_selectivity": 1.0,
"first_query_ms": 20.0,
"recall_at_k": 1.0,
"search": {
"qps": qps,
"latency_ms": {"p50": p50, "p95": p50 * 2, "p99": p50 * 3},
},
}
],
"lifecycle": {
"updates": [
{
"count": 1,
"write_wall_seconds": 0.1,
"next_query_ms": 30.0,
"warm_query_ms": 1.0,
}
],
"delete": {
"write_seconds": 0.1,
"next_query_ms": 30.0,
"warm_query_ms": 1.0,
},
"restart": {
"close_seconds": 0.1,
"adapter_construct_seconds": 0.01,
"first_query_ms": 40.0,
"warm_query_ms": 1.0,
},
},
}
],
}
def write_result(path, document):
path.write_text(json.dumps(document))
return path
def test_summarize_collection_runs_reports_process_median_and_mad(tmp_path):
documents = [
result_document(qps=100, p50=1.0, timestamp="run-1"),
result_document(qps=110, p50=1.2, timestamp="run-2"),
result_document(qps=300, p50=1.1, timestamp="run-3"),
]
for document, latency_ms in zip(documents, (2.0, 2.4, 8.0), strict=True):
document["results"][0]["prebuild_selective_query"] = {
"name": "prebuild_selective_0_1pct",
"filter": {"op": "must", "field": "uniform_bucket", "conds": [0]},
"distribution": "uniform",
"target_selectivity": 0.001,
"latency_ms": latency_ms,
"result_count": 10,
"gpu_delta_bytes": 0,
}
paths = [
write_result(tmp_path / f"run-{index}.json", document)
for index, document in enumerate(documents, start=1)
]
summary = summary_module.summarize_files(paths)
assert summary["run_count"] == 3
assert str(tmp_path) not in json.dumps(summary)
result = summary["results"][0]
assert result["backend"] == "cuvs_brute_force"
assert result["prebuild_selective_query"]["metrics"]["latency_ms"]["median"] == 2.4
assert result["prebuild_selective_query"]["metrics"]["latency_ms"]["mad"] == pytest.approx(0.4)
assert result["searches"][0]["metrics"]["qps"]["median"] == 110
assert result["searches"][0]["metrics"]["qps"]["mad"] == 10
assert result["searches"][0]["metrics"]["warm_p50_ms"]["median"] == pytest.approx(1.1)
assert result["lifecycle"]["updates"][0]["metrics"]["next_query_ms"]["median"] == 30
def test_summarize_collection_runs_rejects_mismatched_dataset(tmp_path):
paths = [
write_result(tmp_path / "run-1.json", result_document(qps=100, p50=1.0, timestamp="run-1")),
write_result(
tmp_path / "run-2.json",
result_document(qps=110, p50=1.2, timestamp="run-2", dimension=16),
),
]
with pytest.raises(ValueError, match="Dataset metadata differs"):
summary_module.summarize_files(paths)
def test_summarize_collection_runs_rejects_duplicate_process(tmp_path):
document = result_document(qps=100, p50=1.0, timestamp="same-run")
paths = [
write_result(tmp_path / "run-1.json", document),
write_result(tmp_path / "renamed-run-1.json", document),
]
with pytest.raises(ValueError, match="Duplicate process timestamp"):
summary_module.summarize_files(paths)