~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
High-value tests for benchmark schema definitions.
Covers:
- BenchmarkStatus enum values and completeness
- DatasetType enum values and completeness
- Table definition structures (columns, indexes, constraints, foreign keys)
- create_benchmark_tables_simple: actual table creation with in-memory SQLite
"""
import pytest
from sqlalchemy import create_engine, inspect
from local_deep_research.web.database.benchmark_schema import (
BenchmarkStatus,
DatasetType,
benchmark_configs_table,
benchmark_progress_table,
benchmark_results_table,
benchmark_runs_table,
create_benchmark_tables_simple,
)
# ---------------------------------------------------------------------------
# Enum definitions
# ---------------------------------------------------------------------------
class TestBenchmarkStatus:
"""Tests for BenchmarkStatus enum."""
def test_all_expected_values(self):
assert BenchmarkStatus.PENDING.value == "pending"
assert BenchmarkStatus.IN_PROGRESS.value == "in_progress"
assert BenchmarkStatus.COMPLETED.value == "completed"
assert BenchmarkStatus.FAILED.value == "failed"
assert BenchmarkStatus.CANCELLED.value == "cancelled"
assert BenchmarkStatus.PAUSED.value == "paused"
def test_member_count(self):
assert len(BenchmarkStatus) == 6
def test_values_are_unique(self):
values = [s.value for s in BenchmarkStatus]
assert len(values) == len(set(values))
def test_lookup_by_name(self):
assert BenchmarkStatus["PENDING"] is BenchmarkStatus.PENDING
def test_lookup_invalid_raises(self):
with pytest.raises(KeyError):
BenchmarkStatus["NONEXISTENT"]
class TestDatasetType:
"""Tests for DatasetType enum."""
def test_all_expected_values(self):
assert DatasetType.SIMPLEQA.value == "simpleqa"
assert DatasetType.BROWSECOMP.value == "browsecomp"
assert DatasetType.CUSTOM.value == "custom"
def test_member_count(self):
assert len(DatasetType) == 3
def test_values_are_unique(self):
values = [d.value for d in DatasetType]
assert len(values) == len(set(values))
# ---------------------------------------------------------------------------
# Table definition structures
# ---------------------------------------------------------------------------
class TestTableDefinitions:
"""Tests for table definition dict structures."""
def test_benchmark_runs_table_name(self):
assert benchmark_runs_table["table_name"] == "benchmark_runs"
def test_benchmark_runs_has_columns(self):
assert len(benchmark_runs_table["columns"]) > 0
def test_benchmark_runs_has_indexes(self):
assert len(benchmark_runs_table["indexes"]) >= 2
def test_benchmark_results_table_name(self):
assert benchmark_results_table["table_name"] == "benchmark_results"
def test_benchmark_results_has_foreign_key(self):
"""benchmark_results has a FK to benchmark_runs.id."""
columns = benchmark_results_table["columns"]
fk_columns = [
c
for c in columns
if c.foreign_keys or (hasattr(c, "name") and "run_id" in str(c))
]
# At least one column should reference benchmark_runs
assert len(fk_columns) >= 1
def test_benchmark_results_has_unique_constraint(self):
constraints = benchmark_results_table.get("constraints", [])
assert len(constraints) >= 1
def test_benchmark_configs_table_name(self):
assert benchmark_configs_table["table_name"] == "benchmark_configs"
def test_benchmark_progress_table_name(self):
assert benchmark_progress_table["table_name"] == "benchmark_progress"
def test_all_tables_have_id_column(self):
for table_def in [
benchmark_runs_table,
benchmark_results_table,
benchmark_configs_table,
benchmark_progress_table,
]:
column_names = [c.name for c in table_def["columns"]]
assert "id" in column_names, (
f"{table_def['table_name']} missing id column"
)
# ---------------------------------------------------------------------------
# create_benchmark_tables_simple with in-memory SQLite
# ---------------------------------------------------------------------------
@pytest.fixture(scope="module")
def db_engine():
"""Create an in-memory SQLite engine shared by schema tests."""
engine = create_engine("sqlite:///:memory:")
create_benchmark_tables_simple(engine)
yield engine
engine.dispose()
class TestCreateBenchmarkTablesSimple:
"""Tests for actual table creation using SQLite in-memory.
Uses a module-scoped engine because the Column objects in the schema module
are module-level singletons that get bound to a Table on first use and
cannot be re-bound to a different MetaData instance.
"""
def test_creates_all_four_tables(self, db_engine):
inspector = inspect(db_engine)
tables = inspector.get_table_names()
assert "benchmark_runs" in tables
assert "benchmark_results" in tables
assert "benchmark_configs" in tables
assert "benchmark_progress" in tables
def test_benchmark_runs_columns(self, db_engine):
inspector = inspect(db_engine)
columns = {c["name"] for c in inspector.get_columns("benchmark_runs")}
expected = {
"id",
"run_name",
"config_hash",
"query_hash_list",
"search_config",
"evaluation_config",
"datasets_config",
"status",
"created_at",
"updated_at",
"start_time",
"end_time",
"total_examples",
"completed_examples",
"failed_examples",
"overall_accuracy",
"processing_rate",
"error_message",
}
assert expected.issubset(columns)
def test_benchmark_results_columns(self, db_engine):
inspector = inspect(db_engine)
columns = {
c["name"] for c in inspector.get_columns("benchmark_results")
}
expected = {
"id",
"benchmark_run_id",
"example_id",
"query_hash",
"dataset_type",
"question",
"correct_answer",
"response",
"is_correct",
"processing_time",
"confidence",
}
assert expected.issubset(columns)
def test_benchmark_configs_columns(self, db_engine):
inspector = inspect(db_engine)
columns = {
c["name"] for c in inspector.get_columns("benchmark_configs")
}
expected = {
"id",
"name",
"description",
"config_hash",
"search_config",
"evaluation_config",
"datasets_config",
"is_default",
"is_public",
"usage_count",
}
assert expected.issubset(columns)
def test_benchmark_progress_columns(self, db_engine):
inspector = inspect(db_engine)
columns = {
c["name"] for c in inspector.get_columns("benchmark_progress")
}
expected = {
"id",
"benchmark_run_id",
"timestamp",
"completed_examples",
"total_examples",
"overall_accuracy",
}
assert expected.issubset(columns)
def test_benchmark_results_foreign_key_exists(self, db_engine):
inspector = inspect(db_engine)
fks = inspector.get_foreign_keys("benchmark_results")
fk_tables = [fk["referred_table"] for fk in fks]
assert "benchmark_runs" in fk_tables
def test_benchmark_progress_foreign_key_exists(self, db_engine):
inspector = inspect(db_engine)
fks = inspector.get_foreign_keys("benchmark_progress")
fk_tables = [fk["referred_table"] for fk in fks]
assert "benchmark_runs" in fk_tables