~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
remove(benchmarks): drop cross-run result reuse (#4498)
LearningCircuit8e94cb385ea5
diff --git a/changelog.d/4498.removal.md b/changelog.d/4498.removal.mdnew file mode 100644index 0000000000..d6441972bc--- /dev/null+++ b/changelog.d/4498.removal.md@@ -0,0 +1 @@+Removed the benchmark result-reuse feature: new benchmark runs no longer silently import results from previous completed runs with matching search settings. The compatibility check covered only a handful of settings (ignoring evaluation config, dataset versions, and the rest of the configuration), so results produced under different conditions could contaminate a run's accuracy, and the reuse accounting could report more than 100% completion. Every run now researches its own sampled questions and reports only its own results.diff --git a/src/local_deep_research/benchmarks/web_api/benchmark_service.py b/src/local_deep_research/benchmarks/web_api/benchmark_service.pyindex f70db309db..0a9c7b27c6 100644--- a/src/local_deep_research/benchmarks/web_api/benchmark_service.py+++ b/src/local_deep_research/benchmarks/web_api/benchmark_service.py@@ -219,62 +219,6 @@ class BenchmarkService:logger.exception("Error creating benchmark run")raise- def get_existing_results(- self,- config_hash: str,- username: Optional[str] = None,- user_password: Optional[str] = None,- ) -> Dict[str, Dict]:- """Get existing results with compatible configuration."""- from ...database.session_context import get_user_db_session-- with get_user_db_session(username, user_password) as session:- try:- # Find compatible runs- compatible_runs = (- session.query(BenchmarkRun)- .filter(BenchmarkRun.config_hash == config_hash)- .filter(BenchmarkRun.status == BenchmarkStatus.COMPLETED)- .all()- )-- existing_results = {}- for run in compatible_runs:- results = (- session.query(BenchmarkResult)- .filter(BenchmarkResult.benchmark_run_id == run.id)- .filter(- BenchmarkResult.is_correct.isnot(None)- ) # Only completed evaluations- .all()- )-- for result in results:- existing_results[result.query_hash] = {- "id": result.example_id,- "dataset_type": result.dataset_type.value,- "problem": result.question,- "correct_answer": result.correct_answer,- "response": result.response,- "extracted_answer": result.extracted_answer,- "confidence": result.confidence,- "processing_time": result.processing_time,- "sources": result.sources,- "is_correct": result.is_correct,- "graded_confidence": result.graded_confidence,- "grader_response": result.grader_response,- "query_hash": result.query_hash,- }-- logger.info(- f"Found {len(existing_results)} existing results for config hash {config_hash}"- )- return existing_results-- except Exception:- logger.exception("Error loading existing results")- return {}-def start_benchmark(self,benchmark_run_id: int,@@ -331,9 +275,6 @@ class BenchmarkService:"datasets_config": benchmark_run.datasets_config,"search_config": benchmark_run.search_config,"evaluation_config": benchmark_run.evaluation_config,- "existing_results": self.get_existing_results(- benchmark_run.config_hash, username, user_password- ),"settings_snapshot": settings_snapshot, # Add settings snapshot}@@ -419,18 +360,16 @@ class BenchmarkService:datasets_config = benchmark_data["datasets_config"]search_config = benchmark_data["search_config"]evaluation_config = benchmark_data["evaluation_config"]- existing_results = benchmark_data.get("existing_results", {})# Create task queuetask_queue = self._create_task_queue(datasets_config,- existing_results,benchmark_run_id,)# Calculate totals- total_examples = len(task_queue) + len(existing_results)- completed_examples = len(existing_results)+ total_examples = len(task_queue)+ completed_examples = 0# Initialize task trackingtask_id = f"benchmark_{benchmark_run_id}_{int(datetime.now(UTC).timestamp())}"@@ -607,10 +546,9 @@ class BenchmarkService:def _create_task_queue(self,datasets_config: Dict,- existing_results: Dict,benchmark_run_id: int,) -> List[Dict]:- """Create list of tasks to process, excluding existing results."""+ """Create list of tasks to process."""tasks: List[Dict[str, Any]] = []for dataset_name, config in datasets_config.items():@@ -635,10 +573,6 @@ class BenchmarkService:question, dataset_name)- # Skip if already processed- if query_hash in existing_results:- continue-tasks.append({"benchmark_run_id": benchmark_run_id,@@ -1229,9 +1163,8 @@ class BenchmarkService:if not benchmark_run:return None- # Calculate running accuracy from current results AND reused results from compatible runs- # First get results specifically for this benchmark run- current_results = (+ # Calculate running accuracy from this run's evaluated results+ results = (session.query(BenchmarkResult).filter(BenchmarkResult.benchmark_run_id == benchmark_run_id@@ -1240,45 +1173,6 @@ class BenchmarkService:.all())- # Then get reused results from compatible benchmark runs (same config hash)- # Only count results up to the number we say we've "completed"- if benchmark_run.completed_examples > len(current_results):- # We have reused results, get them from compatible runs- reused_count_needed = (- benchmark_run.completed_examples - len(current_results)- )-- compatible_results = (- session.query(BenchmarkResult)- .join(- BenchmarkRun,- BenchmarkResult.benchmark_run_id == BenchmarkRun.id,- )- .filter(- BenchmarkRun.config_hash- == benchmark_run.config_hash- )- .filter(- BenchmarkRun.id != benchmark_run_id- ) # Exclude current run- .filter(- BenchmarkRun.status == BenchmarkStatus.COMPLETED- )- .filter(BenchmarkResult.is_correct.isnot(None))- .order_by(BenchmarkResult.id) # Consistent ordering- .limit(reused_count_needed)- .all()- )-- # Combine current and reused results- results = (- current_results- + compatible_results[:reused_count_needed]- )- else:- # No reused results, just use current results- results = current_results-running_accuracy = None# Dynamic per-dataset accuracy trackingdataset_accuracies = {}diff --git a/tests/benchmarks/test_benchmark_service.py b/tests/benchmarks/test_benchmark_service.pyindex f0c0170c41..4aafa13171 100644--- a/tests/benchmarks/test_benchmark_service.py+++ b/tests/benchmarks/test_benchmark_service.py@@ -1213,7 +1213,6 @@ class TestBenchmarkServiceTaskQueue:tasks = service._create_task_queue(datasets_config=datasets_config,- existing_results={},benchmark_run_id=1,)@@ -1222,133 +1221,6 @@ class TestBenchmarkServiceTaskQueue:assert tasks[0]["correct_answer"] == "A1"assert tasks[0]["benchmark_run_id"] == 1- def test_create_task_queue_excludes_existing_results(self):- """Test that existing results are excluded from task queue."""- from local_deep_research.benchmarks.web_api.benchmark_service import (- BenchmarkService,- )-- mock_socket = Mock()- service = BenchmarkService(socket_service=mock_socket)-- datasets_config = {"simpleqa": {"count": 3}}-- with patch(- "local_deep_research.benchmarks.web_api.benchmark_service.load_dataset"- ) as mock_load:- mock_load.return_value = [- {"id": "1", "problem": "Q1", "answer": "A1"},- {"id": "2", "problem": "Q2", "answer": "A2"},- {"id": "3", "problem": "Q3", "answer": "A3"},- ]-- # Generate the hash for Q2- q2_hash = service.generate_query_hash("Q2", "simpleqa")-- existing_results = {q2_hash: {"id": "2"}}-- tasks = service._create_task_queue(- datasets_config=datasets_config,- existing_results=existing_results,- benchmark_run_id=1,- )-- # Only 2 tasks should be created (Q2 is excluded)- assert len(tasks) == 2- questions = [t["question"] for t in tasks]- assert "Q2" not in questions---class TestBenchmarkServiceGetExistingResults:- """Tests for get_existing_results functionality."""-- def test_get_existing_results_returns_empty_for_no_matches(self):- """Test that get_existing_results returns empty dict when no matches."""- from local_deep_research.benchmarks.web_api.benchmark_service import (- BenchmarkService,- )-- mock_socket = Mock()- service = BenchmarkService(socket_service=mock_socket)-- with patch(- "local_deep_research.database.session_context.get_user_db_session"- ) as mock_get_session:- mock_session = Mock()- mock_session.__enter__ = Mock(return_value=mock_session)- mock_session.__exit__ = Mock(return_value=False)- mock_get_session.return_value = mock_session-- # No compatible runs- mock_session.query.return_value.filter.return_value.filter.return_value.all.return_value = []-- result = service.get_existing_results(- "abc12345", username="testuser"- )-- assert result == {}-- def test_get_existing_results_finds_compatible_results(self):- """Test that get_existing_results finds results from compatible runs."""- from local_deep_research.benchmarks.web_api.benchmark_service import (- BenchmarkService,- )- from local_deep_research.database.models.benchmark import DatasetType-- mock_socket = Mock()- service = BenchmarkService(socket_service=mock_socket)-- with patch(- "local_deep_research.database.session_context.get_user_db_session"- ) as mock_get_session:- mock_session = Mock()- mock_session.__enter__ = Mock(return_value=mock_session)- mock_session.__exit__ = Mock(return_value=False)- mock_get_session.return_value = mock_session-- # Mock a compatible run- mock_run = Mock()- mock_run.id = 1-- # Mock existing results- mock_result = Mock()- mock_result.query_hash = "hash123"- mock_result.example_id = "ex1"- mock_result.dataset_type = DatasetType.SIMPLEQA- mock_result.question = "What is 2+2?"- mock_result.correct_answer = "4"- mock_result.response = "4"- mock_result.extracted_answer = "4"- mock_result.confidence = "100"- mock_result.processing_time = 1.5- mock_result.sources = "[]"- mock_result.is_correct = True- mock_result.graded_confidence = "100"- mock_result.grader_response = "Correct"-- # Setup query chain- def query_side_effect(model):- if "BenchmarkRun" in str(model):- mock_q = Mock()- mock_q.filter.return_value.filter.return_value.all.return_value = [- mock_run- ]- return mock_q- mock_q = Mock()- mock_q.filter.return_value.filter.return_value.all.return_value = [- mock_result- ]- return mock_q-- mock_session.query.side_effect = query_side_effect-- result = service.get_existing_results(- "abc12345", username="testuser"- )-- assert "hash123" in result- assert result["hash123"]["is_correct"] is True-class TestBenchmarkServiceUpdateStatus:"""Tests for update_benchmark_status functionality."""diff --git a/tests/benchmarks/web_api/test_benchmark_service_coverage.py b/tests/benchmarks/web_api/test_benchmark_service_coverage.pyindex 27560f98e6..04dbea913f 100644--- a/tests/benchmarks/web_api/test_benchmark_service_coverage.py+++ b/tests/benchmarks/web_api/test_benchmark_service_coverage.py@@ -301,74 +301,6 @@ class TestCreateBenchmarkRun:mock_session.rollback.assert_called_once()-# ============================================================-# get_existing_results-# ============================================================---class TestGetExistingResults:- def test_get_existing_results_success(self):- svc = _make_service()- mock_session = MagicMock()-- mock_run = MagicMock()- mock_run.id = 1-- mock_result = MagicMock()- mock_result.query_hash = "hash1"- mock_result.example_id = "ex1"- mock_result.dataset_type.value = "simpleqa"- mock_result.question = "Q?"- mock_result.correct_answer = "A"- mock_result.response = "resp"- mock_result.extracted_answer = "ans"- mock_result.confidence = "90"- mock_result.processing_time = 5.0- mock_result.sources = "[]"- mock_result.is_correct = True- mock_result.graded_confidence = "95"- mock_result.grader_response = "correct"-- # Chain the query- mock_session.query.return_value.filter.return_value.filter.return_value.all.return_value = [- mock_run- ]- # Second query for results- mock_session.query.return_value.filter.return_value.filter.return_value.all.side_effect = [- [mock_run],- [mock_result],- ]-- with patch(- "local_deep_research.database.session_context.get_user_db_session"- ) as mock_get_session:- mock_get_session.return_value.__enter__ = Mock(- return_value=mock_session- )- mock_get_session.return_value.__exit__ = Mock(return_value=False)-- results = svc.get_existing_results("hash123", "user1")-- assert "hash1" in results-- def test_get_existing_results_exception(self):- svc = _make_service()- mock_session = MagicMock()- mock_session.query.side_effect = RuntimeError("db error")-- with patch(- "local_deep_research.database.session_context.get_user_db_session"- ) as mock_get_session:- mock_get_session.return_value.__enter__ = Mock(- return_value=mock_session- )- mock_get_session.return_value.__exit__ = Mock(return_value=False)-- results = svc.get_existing_results("hash123")-- assert results == {}--# ============================================================# _create_task_queue# ============================================================@@ -385,7 +317,6 @@ class TestCreateTaskQueue:tasks = svc._create_task_queue({"simpleqa": {"count": 2}},- existing_results={},benchmark_run_id=1,)@@ -394,30 +325,36 @@ class TestCreateTaskQueue:assert tasks[0]["dataset_type"] == "simpleqa"@patch(f"{MODULE}.load_dataset")- def test_skips_existing_results(self, mock_load):+ def test_queues_every_sampled_question(self, mock_load):+ """Regression test for #4498: every sampled question becomes a task.++ The removed cross-run reuse feature silently skipped questions that+ had results in previous compatible runs, which broke the+ completed/total accounting (#4451). A run must always process its+ full sample, even if identical questions were answered before."""svc = _make_service()examples = [- {"id": "ex1", "problem": "Q1?", "answer": "A1"},+ {"id": f"ex{i}", "problem": f"Q{i}?", "answer": f"A{i}"}+ for i in range(5)]mock_load.return_value = examples- # Pre-compute the hash for Q1?- query_hash = svc.generate_query_hash("Q1?", "simpleqa")-tasks = svc._create_task_queue(- {"simpleqa": {"count": 1}},- existing_results={query_hash: {"some": "data"}},+ {"simpleqa": {"count": 5}},benchmark_run_id=1,)- assert len(tasks) == 0+ assert len(tasks) == len(examples)+ assert [t["question"] for t in tasks] == [+ e["problem"] for e in examples+ ]+ assert [t["task_index"] for t in tasks] == list(range(5))@patch(f"{MODULE}.load_dataset")def test_skips_zero_count_datasets(self, mock_load):svc = _make_service()tasks = svc._create_task_queue({"simpleqa": {"count": 0}},- existing_results={},benchmark_run_id=1,)assert len(tasks) == 0@@ -432,7 +369,6 @@ class TestCreateTaskQueue:tasks = svc._create_task_queue({"browsecomp": {"count": 1}},- existing_results={},benchmark_run_id=1,)@@ -448,7 +384,6 @@ class TestCreateTaskQueue:tasks = svc._create_task_queue({"simpleqa": {"count": 1}},- existing_results={},benchmark_run_id=1,)@@ -1482,7 +1417,6 @@ class TestRunBenchmarkThread:"datasets_config": {"simpleqa": {"count": 1}},"search_config": {"iterations": 1},"evaluation_config": {},- "existing_results": {},"settings_snapshot": {},},"results": [],@@ -1525,7 +1459,6 @@ class TestRunBenchmarkThread:"datasets_config": {},"search_config": {},"evaluation_config": {},- "existing_results": {},"settings_snapshot": {},},"status": "cancelled",@@ -1569,7 +1502,6 @@ class TestRunBenchmarkThread:"datasets_config": {},"search_config": {},"evaluation_config": {},- "existing_results": {},"settings_snapshot": {},},"results": [],@@ -1627,7 +1559,6 @@ class TestRunBenchmarkThread:"datasets_config": {},"search_config": {},"evaluation_config": {},- "existing_results": {},"settings_snapshot": {"key1": {"value": "val1"},"key2": "direct_val",@@ -1669,7 +1600,6 @@ class TestRunBenchmarkThread:"datasets_config": {},"search_config": {},"evaluation_config": {},- "existing_results": {},"settings_snapshot": {},},"results": [],