Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

remove(benchmarks): drop cross-run result reuse (#4498)

LearningCircuit8e94cb385ea5
diff --git a/changelog.d/4498.removal.md b/changelog.d/4498.removal.md
new file mode 100644
index 0000000000..d6441972bc
--- /dev/null
+++ b/changelog.d/4498.removal.md
@@ -0,0 +1 @@
+Removed the benchmark result-reuse feature: new benchmark runs no longer silently import results from previous completed runs with matching search settings. The compatibility check covered only a handful of settings (ignoring evaluation config, dataset versions, and the rest of the configuration), so results produced under different conditions could contaminate a run's accuracy, and the reuse accounting could report more than 100% completion. Every run now researches its own sampled questions and reports only its own results.
diff --git a/src/local_deep_research/benchmarks/web_api/benchmark_service.py b/src/local_deep_research/benchmarks/web_api/benchmark_service.py
index f70db309db..0a9c7b27c6 100644
--- a/src/local_deep_research/benchmarks/web_api/benchmark_service.py
+++ b/src/local_deep_research/benchmarks/web_api/benchmark_service.py
@@ -219,62 +219,6 @@ class BenchmarkService:
logger.exception("Error creating benchmark run")
raise
- def get_existing_results(
- self,
- config_hash: str,
- username: Optional[str] = None,
- user_password: Optional[str] = None,
- ) -> Dict[str, Dict]:
- """Get existing results with compatible configuration."""
- from ...database.session_context import get_user_db_session
-
- with get_user_db_session(username, user_password) as session:
- try:
- # Find compatible runs
- compatible_runs = (
- session.query(BenchmarkRun)
- .filter(BenchmarkRun.config_hash == config_hash)
- .filter(BenchmarkRun.status == BenchmarkStatus.COMPLETED)
- .all()
- )
-
- existing_results = {}
- for run in compatible_runs:
- results = (
- session.query(BenchmarkResult)
- .filter(BenchmarkResult.benchmark_run_id == run.id)
- .filter(
- BenchmarkResult.is_correct.isnot(None)
- ) # Only completed evaluations
- .all()
- )
-
- for result in results:
- existing_results[result.query_hash] = {
- "id": result.example_id,
- "dataset_type": result.dataset_type.value,
- "problem": result.question,
- "correct_answer": result.correct_answer,
- "response": result.response,
- "extracted_answer": result.extracted_answer,
- "confidence": result.confidence,
- "processing_time": result.processing_time,
- "sources": result.sources,
- "is_correct": result.is_correct,
- "graded_confidence": result.graded_confidence,
- "grader_response": result.grader_response,
- "query_hash": result.query_hash,
- }
-
- logger.info(
- f"Found {len(existing_results)} existing results for config hash {config_hash}"
- )
- return existing_results
-
- except Exception:
- logger.exception("Error loading existing results")
- return {}
-
def start_benchmark(
self,
benchmark_run_id: int,
@@ -331,9 +275,6 @@ class BenchmarkService:
"datasets_config": benchmark_run.datasets_config,
"search_config": benchmark_run.search_config,
"evaluation_config": benchmark_run.evaluation_config,
- "existing_results": self.get_existing_results(
- benchmark_run.config_hash, username, user_password
- ),
"settings_snapshot": settings_snapshot, # Add settings snapshot
}
@@ -419,18 +360,16 @@ class BenchmarkService:
datasets_config = benchmark_data["datasets_config"]
search_config = benchmark_data["search_config"]
evaluation_config = benchmark_data["evaluation_config"]
- existing_results = benchmark_data.get("existing_results", {})
# Create task queue
task_queue = self._create_task_queue(
datasets_config,
- existing_results,
benchmark_run_id,
)
# Calculate totals
- total_examples = len(task_queue) + len(existing_results)
- completed_examples = len(existing_results)
+ total_examples = len(task_queue)
+ completed_examples = 0
# Initialize task tracking
task_id = f"benchmark_{benchmark_run_id}_{int(datetime.now(UTC).timestamp())}"
@@ -607,10 +546,9 @@ class BenchmarkService:
def _create_task_queue(
self,
datasets_config: Dict,
- existing_results: Dict,
benchmark_run_id: int,
) -> List[Dict]:
- """Create list of tasks to process, excluding existing results."""
+ """Create list of tasks to process."""
tasks: List[Dict[str, Any]] = []
for dataset_name, config in datasets_config.items():
@@ -635,10 +573,6 @@ class BenchmarkService:
question, dataset_name
)
- # Skip if already processed
- if query_hash in existing_results:
- continue
-
tasks.append(
{
"benchmark_run_id": benchmark_run_id,
@@ -1229,9 +1163,8 @@ class BenchmarkService:
if not benchmark_run:
return None
- # Calculate running accuracy from current results AND reused results from compatible runs
- # First get results specifically for this benchmark run
- current_results = (
+ # Calculate running accuracy from this run's evaluated results
+ results = (
session.query(BenchmarkResult)
.filter(
BenchmarkResult.benchmark_run_id == benchmark_run_id
@@ -1240,45 +1173,6 @@ class BenchmarkService:
.all()
)
- # Then get reused results from compatible benchmark runs (same config hash)
- # Only count results up to the number we say we've "completed"
- if benchmark_run.completed_examples > len(current_results):
- # We have reused results, get them from compatible runs
- reused_count_needed = (
- benchmark_run.completed_examples - len(current_results)
- )
-
- compatible_results = (
- session.query(BenchmarkResult)
- .join(
- BenchmarkRun,
- BenchmarkResult.benchmark_run_id == BenchmarkRun.id,
- )
- .filter(
- BenchmarkRun.config_hash
- == benchmark_run.config_hash
- )
- .filter(
- BenchmarkRun.id != benchmark_run_id
- ) # Exclude current run
- .filter(
- BenchmarkRun.status == BenchmarkStatus.COMPLETED
- )
- .filter(BenchmarkResult.is_correct.isnot(None))
- .order_by(BenchmarkResult.id) # Consistent ordering
- .limit(reused_count_needed)
- .all()
- )
-
- # Combine current and reused results
- results = (
- current_results
- + compatible_results[:reused_count_needed]
- )
- else:
- # No reused results, just use current results
- results = current_results
-
running_accuracy = None
# Dynamic per-dataset accuracy tracking
dataset_accuracies = {}
diff --git a/tests/benchmarks/test_benchmark_service.py b/tests/benchmarks/test_benchmark_service.py
index f0c0170c41..4aafa13171 100644
--- a/tests/benchmarks/test_benchmark_service.py
+++ b/tests/benchmarks/test_benchmark_service.py
@@ -1213,7 +1213,6 @@ class TestBenchmarkServiceTaskQueue:
tasks = service._create_task_queue(
datasets_config=datasets_config,
- existing_results={},
benchmark_run_id=1,
)
@@ -1222,133 +1221,6 @@ class TestBenchmarkServiceTaskQueue:
assert tasks[0]["correct_answer"] == "A1"
assert tasks[0]["benchmark_run_id"] == 1
- def test_create_task_queue_excludes_existing_results(self):
- """Test that existing results are excluded from task queue."""
- from local_deep_research.benchmarks.web_api.benchmark_service import (
- BenchmarkService,
- )
-
- mock_socket = Mock()
- service = BenchmarkService(socket_service=mock_socket)
-
- datasets_config = {"simpleqa": {"count": 3}}
-
- with patch(
- "local_deep_research.benchmarks.web_api.benchmark_service.load_dataset"
- ) as mock_load:
- mock_load.return_value = [
- {"id": "1", "problem": "Q1", "answer": "A1"},
- {"id": "2", "problem": "Q2", "answer": "A2"},
- {"id": "3", "problem": "Q3", "answer": "A3"},
- ]
-
- # Generate the hash for Q2
- q2_hash = service.generate_query_hash("Q2", "simpleqa")
-
- existing_results = {q2_hash: {"id": "2"}}
-
- tasks = service._create_task_queue(
- datasets_config=datasets_config,
- existing_results=existing_results,
- benchmark_run_id=1,
- )
-
- # Only 2 tasks should be created (Q2 is excluded)
- assert len(tasks) == 2
- questions = [t["question"] for t in tasks]
- assert "Q2" not in questions
-
-
-class TestBenchmarkServiceGetExistingResults:
- """Tests for get_existing_results functionality."""
-
- def test_get_existing_results_returns_empty_for_no_matches(self):
- """Test that get_existing_results returns empty dict when no matches."""
- from local_deep_research.benchmarks.web_api.benchmark_service import (
- BenchmarkService,
- )
-
- mock_socket = Mock()
- service = BenchmarkService(socket_service=mock_socket)
-
- with patch(
- "local_deep_research.database.session_context.get_user_db_session"
- ) as mock_get_session:
- mock_session = Mock()
- mock_session.__enter__ = Mock(return_value=mock_session)
- mock_session.__exit__ = Mock(return_value=False)
- mock_get_session.return_value = mock_session
-
- # No compatible runs
- mock_session.query.return_value.filter.return_value.filter.return_value.all.return_value = []
-
- result = service.get_existing_results(
- "abc12345", username="testuser"
- )
-
- assert result == {}
-
- def test_get_existing_results_finds_compatible_results(self):
- """Test that get_existing_results finds results from compatible runs."""
- from local_deep_research.benchmarks.web_api.benchmark_service import (
- BenchmarkService,
- )
- from local_deep_research.database.models.benchmark import DatasetType
-
- mock_socket = Mock()
- service = BenchmarkService(socket_service=mock_socket)
-
- with patch(
- "local_deep_research.database.session_context.get_user_db_session"
- ) as mock_get_session:
- mock_session = Mock()
- mock_session.__enter__ = Mock(return_value=mock_session)
- mock_session.__exit__ = Mock(return_value=False)
- mock_get_session.return_value = mock_session
-
- # Mock a compatible run
- mock_run = Mock()
- mock_run.id = 1
-
- # Mock existing results
- mock_result = Mock()
- mock_result.query_hash = "hash123"
- mock_result.example_id = "ex1"
- mock_result.dataset_type = DatasetType.SIMPLEQA
- mock_result.question = "What is 2+2?"
- mock_result.correct_answer = "4"
- mock_result.response = "4"
- mock_result.extracted_answer = "4"
- mock_result.confidence = "100"
- mock_result.processing_time = 1.5
- mock_result.sources = "[]"
- mock_result.is_correct = True
- mock_result.graded_confidence = "100"
- mock_result.grader_response = "Correct"
-
- # Setup query chain
- def query_side_effect(model):
- if "BenchmarkRun" in str(model):
- mock_q = Mock()
- mock_q.filter.return_value.filter.return_value.all.return_value = [
- mock_run
- ]
- return mock_q
- mock_q = Mock()
- mock_q.filter.return_value.filter.return_value.all.return_value = [
- mock_result
- ]
- return mock_q
-
- mock_session.query.side_effect = query_side_effect
-
- result = service.get_existing_results(
- "abc12345", username="testuser"
- )
-
- assert "hash123" in result
- assert result["hash123"]["is_correct"] is True
-
class TestBenchmarkServiceUpdateStatus:
"""Tests for update_benchmark_status functionality."""
diff --git a/tests/benchmarks/web_api/test_benchmark_service_coverage.py b/tests/benchmarks/web_api/test_benchmark_service_coverage.py
index 27560f98e6..04dbea913f 100644
--- a/tests/benchmarks/web_api/test_benchmark_service_coverage.py
+++ b/tests/benchmarks/web_api/test_benchmark_service_coverage.py
@@ -301,74 +301,6 @@ class TestCreateBenchmarkRun:
mock_session.rollback.assert_called_once()
-# ============================================================
-# get_existing_results
-# ============================================================
-
-
-class TestGetExistingResults:
- def test_get_existing_results_success(self):
- svc = _make_service()
- mock_session = MagicMock()
-
- mock_run = MagicMock()
- mock_run.id = 1
-
- mock_result = MagicMock()
- mock_result.query_hash = "hash1"
- mock_result.example_id = "ex1"
- mock_result.dataset_type.value = "simpleqa"
- mock_result.question = "Q?"
- mock_result.correct_answer = "A"
- mock_result.response = "resp"
- mock_result.extracted_answer = "ans"
- mock_result.confidence = "90"
- mock_result.processing_time = 5.0
- mock_result.sources = "[]"
- mock_result.is_correct = True
- mock_result.graded_confidence = "95"
- mock_result.grader_response = "correct"
-
- # Chain the query
- mock_session.query.return_value.filter.return_value.filter.return_value.all.return_value = [
- mock_run
- ]
- # Second query for results
- mock_session.query.return_value.filter.return_value.filter.return_value.all.side_effect = [
- [mock_run],
- [mock_result],
- ]
-
- with patch(
- "local_deep_research.database.session_context.get_user_db_session"
- ) as mock_get_session:
- mock_get_session.return_value.__enter__ = Mock(
- return_value=mock_session
- )
- mock_get_session.return_value.__exit__ = Mock(return_value=False)
-
- results = svc.get_existing_results("hash123", "user1")
-
- assert "hash1" in results
-
- def test_get_existing_results_exception(self):
- svc = _make_service()
- mock_session = MagicMock()
- mock_session.query.side_effect = RuntimeError("db error")
-
- with patch(
- "local_deep_research.database.session_context.get_user_db_session"
- ) as mock_get_session:
- mock_get_session.return_value.__enter__ = Mock(
- return_value=mock_session
- )
- mock_get_session.return_value.__exit__ = Mock(return_value=False)
-
- results = svc.get_existing_results("hash123")
-
- assert results == {}
-
-
# ============================================================
# _create_task_queue
# ============================================================
@@ -385,7 +317,6 @@ class TestCreateTaskQueue:
tasks = svc._create_task_queue(
{"simpleqa": {"count": 2}},
- existing_results={},
benchmark_run_id=1,
)
@@ -394,30 +325,36 @@ class TestCreateTaskQueue:
assert tasks[0]["dataset_type"] == "simpleqa"
@patch(f"{MODULE}.load_dataset")
- def test_skips_existing_results(self, mock_load):
+ def test_queues_every_sampled_question(self, mock_load):
+ """Regression test for #4498: every sampled question becomes a task.
+
+ The removed cross-run reuse feature silently skipped questions that
+ had results in previous compatible runs, which broke the
+ completed/total accounting (#4451). A run must always process its
+ full sample, even if identical questions were answered before."""
svc = _make_service()
examples = [
- {"id": "ex1", "problem": "Q1?", "answer": "A1"},
+ {"id": f"ex{i}", "problem": f"Q{i}?", "answer": f"A{i}"}
+ for i in range(5)
]
mock_load.return_value = examples
- # Pre-compute the hash for Q1?
- query_hash = svc.generate_query_hash("Q1?", "simpleqa")
-
tasks = svc._create_task_queue(
- {"simpleqa": {"count": 1}},
- existing_results={query_hash: {"some": "data"}},
+ {"simpleqa": {"count": 5}},
benchmark_run_id=1,
)
- assert len(tasks) == 0
+ assert len(tasks) == len(examples)
+ assert [t["question"] for t in tasks] == [
+ e["problem"] for e in examples
+ ]
+ assert [t["task_index"] for t in tasks] == list(range(5))
@patch(f"{MODULE}.load_dataset")
def test_skips_zero_count_datasets(self, mock_load):
svc = _make_service()
tasks = svc._create_task_queue(
{"simpleqa": {"count": 0}},
- existing_results={},
benchmark_run_id=1,
)
assert len(tasks) == 0
@@ -432,7 +369,6 @@ class TestCreateTaskQueue:
tasks = svc._create_task_queue(
{"browsecomp": {"count": 1}},
- existing_results={},
benchmark_run_id=1,
)
@@ -448,7 +384,6 @@ class TestCreateTaskQueue:
tasks = svc._create_task_queue(
{"simpleqa": {"count": 1}},
- existing_results={},
benchmark_run_id=1,
)
@@ -1482,7 +1417,6 @@ class TestRunBenchmarkThread:
"datasets_config": {"simpleqa": {"count": 1}},
"search_config": {"iterations": 1},
"evaluation_config": {},
- "existing_results": {},
"settings_snapshot": {},
},
"results": [],
@@ -1525,7 +1459,6 @@ class TestRunBenchmarkThread:
"datasets_config": {},
"search_config": {},
"evaluation_config": {},
- "existing_results": {},
"settings_snapshot": {},
},
"status": "cancelled",
@@ -1569,7 +1502,6 @@ class TestRunBenchmarkThread:
"datasets_config": {},
"search_config": {},
"evaluation_config": {},
- "existing_results": {},
"settings_snapshot": {},
},
"results": [],
@@ -1627,7 +1559,6 @@ class TestRunBenchmarkThread:
"datasets_config": {},
"search_config": {},
"evaluation_config": {},
- "existing_results": {},
"settings_snapshot": {
"key1": {"value": "val1"},
"key2": "direct_val",
@@ -1669,7 +1600,6 @@ class TestRunBenchmarkThread:
"datasets_config": {},
"search_config": {},
"evaluation_config": {},
- "existing_results": {},
"settings_snapshot": {},
},
"results": [],