~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(benchmarks): include example_id in query hash so distinct examples persist (#5080)
Ehsan178eddc2bf2f
diff --git a/changelog.d/4860.bugfix.md b/changelog.d/4860.bugfix.mdnew file mode 100644index 0000000000..c2d884d176--- /dev/null+++ b/changelog.d/4860.bugfix.md@@ -0,0 +1 @@+Benchmark runs now persist one result row per example even when two examples have identical question text. Previously the per-query hash keyed only on the question and dataset, so distinct examples with the same question collided on the `uix_run_query` unique constraint and all but the first were dropped; the stored row count fell below the number of examples executed and disagreed with the in-memory accuracy. The example id is now part of the hash, so each example persists its own row while a genuine re-sync of the same example is still deduplicated.diff --git a/src/local_deep_research/benchmarks/web_api/benchmark_service.py b/src/local_deep_research/benchmarks/web_api/benchmark_service.pyindex b3aedc85a7..eff9aedf06 100644--- a/src/local_deep_research/benchmarks/web_api/benchmark_service.py+++ b/src/local_deep_research/benchmarks/web_api/benchmark_service.py@@ -186,9 +186,20 @@ class BenchmarkService:config_str.encode(), usedforsecurity=False).hexdigest()[:8]- def generate_query_hash(self, question: str, dataset_type: str) -> str:- """Generate a hash for a query to enable deduplication."""- query_content = f"{question.strip()}|{dataset_type.lower()}"+ def generate_query_hash(+ self, question: str, dataset_type: str, example_id: str+ ) -> str:+ """Generate a hash for a query to enable deduplication.++ The example id is part of the key so two distinct examples that share+ the same question text (and dataset) get distinct hashes and each+ persist a row, instead of colliding on the ``uix_run_query`` unique+ constraint and dropping one. Re-hashing the same example is stable, so+ the sync dedup still collapses a genuine re-sync of that example.+ """+ query_content = (+ f"{example_id}|{question.strip()}|{dataset_type.lower()}"+ )return hashlib.md5( # DevSkim: ignore DS126858query_content.encode(), usedforsecurity=False).hexdigest()@@ -626,16 +637,18 @@ class BenchmarkService:# All registered datasets expose "problem"/"answer"question = example.get("problem", "")correct_answer = example.get("answer", "")+ example_id = example.get("id", f"example_{i}")- # Generate query hash+ # Generate query hash (keyed on the example so distinct+ # examples with identical question text don't collide)query_hash = self.generate_query_hash(- question, dataset_name+ question, dataset_name, example_id)tasks.append({"benchmark_run_id": benchmark_run_id,- "example_id": example.get("id", f"example_{i}"),+ "example_id": example_id,"dataset_type": dataset_name,"question": question,"correct_answer": correct_answer,diff --git a/tests/benchmarks/test_benchmark_service.py b/tests/benchmarks/test_benchmark_service.pyindex d5750a8e2c..6b3105a68c 100644--- a/tests/benchmarks/test_benchmark_service.py+++ b/tests/benchmarks/test_benchmark_service.py@@ -377,13 +377,39 @@ class TestBenchmarkServiceQueryHash:mock_socket.return_value = Mock()service = BenchmarkService()- hash1 = service.generate_query_hash("What is AI?", "simpleqa")- hash2 = service.generate_query_hash("What is AI?", "simpleqa")+ hash1 = service.generate_query_hash(+ "What is AI?", "simpleqa", "ex1"+ )+ hash2 = service.generate_query_hash(+ "What is AI?", "simpleqa", "ex1"+ )- # Same question and dataset should produce same hash+ # Same question, dataset and example should produce same hashassert hash1 == hash2assert len(hash1) == 32 # MD5 hex digest+ def test_different_example_ids_different_hashes(self):+ """#4860: distinct examples with identical question text must not+ collide, otherwise the sync dedup drops all but the first row."""+ from local_deep_research.benchmarks.web_api.benchmark_service import (+ BenchmarkService,+ )++ with patch(+ "local_deep_research.benchmarks.web_api.benchmark_service.SocketIOService"+ ) as mock_socket:+ mock_socket.return_value = Mock()+ service = BenchmarkService()++ hash1 = service.generate_query_hash(+ "Same question", "simpleqa", "ex1"+ )+ hash2 = service.generate_query_hash(+ "Same question", "simpleqa", "ex2"+ )++ assert hash1 != hash2+def test_different_questions_different_hashes(self):"""Test that different questions produce different hashes."""from local_deep_research.benchmarks.web_api.benchmark_service import (@@ -396,8 +422,8 @@ class TestBenchmarkServiceQueryHash:mock_socket.return_value = Mock()service = BenchmarkService()- hash1 = service.generate_query_hash("Question 1", "simpleqa")- hash2 = service.generate_query_hash("Question 2", "simpleqa")+ hash1 = service.generate_query_hash("Question 1", "simpleqa", "ex1")+ hash2 = service.generate_query_hash("Question 2", "simpleqa", "ex1")assert hash1 != hash2@@ -413,8 +439,12 @@ class TestBenchmarkServiceQueryHash:mock_socket.return_value = Mock()service = BenchmarkService()- hash1 = service.generate_query_hash("Same question", "simpleqa")- hash2 = service.generate_query_hash("Same question", "browsecomp")+ hash1 = service.generate_query_hash(+ "Same question", "simpleqa", "ex1"+ )+ hash2 = service.generate_query_hash(+ "Same question", "browsecomp", "ex1"+ )assert hash1 != hash2@@ -430,8 +460,10 @@ class TestBenchmarkServiceQueryHash:mock_socket.return_value = Mock()service = BenchmarkService()- hash1 = service.generate_query_hash(" Question ", "simpleqa")- hash2 = service.generate_query_hash("Question", "simpleqa")+ hash1 = service.generate_query_hash(+ " Question ", "simpleqa", "ex1"+ )+ hash2 = service.generate_query_hash("Question", "simpleqa", "ex1")assert hash1 == hash2diff --git a/tests/benchmarks/web_api/test_benchmark_result_dedup.py b/tests/benchmarks/web_api/test_benchmark_result_dedup.pyindex f4dd8ef080..7ce70e807d 100644--- a/tests/benchmarks/web_api/test_benchmark_result_dedup.py+++ b/tests/benchmarks/web_api/test_benchmark_result_dedup.py@@ -431,3 +431,41 @@ def test_callers_do_not_mark_saved_on_commit_failure():with patch(target, _mock_session_cm(mock_session)):svc._sync_results_to_database(2)assert "saved_indices" not in run2, run2.get("saved_indices")+++# ---------------------------------------------------------------------------+# #4860: the query_hash must key on the example, not just the question text+# ---------------------------------------------------------------------------+++def test_distinct_examples_same_question_persist_separately():+ """#4860: two DISTINCT examples that share identical question text (and+ dataset) must each persist a row.++ ``generate_query_hash`` used to key only on ``question`` + ``dataset``, so+ two different examples with the same question produced the SAME hash and+ the sync dedup (``uix_run_query``) dropped the second, so the stored row+ count fell below the number of examples executed. Driven end-to-end+ through the real ``_create_task_queue`` (which computes the hash) and+ ``_persist_unsaved_results`` (which enforces the constraint)."""+ svc = BenchmarkService(socket_service=MagicMock())+ session = _make_session()++ dataset = [+ {"id": "ex_A", "problem": "Capital of France?", "answer": "Paris"},+ {"id": "ex_B", "problem": "Capital of France?", "answer": "Paris"},+ ]+ with patch(+ "local_deep_research.benchmarks.web_api.benchmark_service.load_dataset",+ return_value=dataset,+ ):+ tasks = svc._create_task_queue({"simpleqa": {"count": 2}}, 1)++ # Distinct examples -> distinct hashes, even with identical question text.+ assert len({t["query_hash"] for t in tasks}) == 2++ staged = svc._persist_unsaved_results(session, 1, {"results": tasks})+ session.commit() # both rows land; before the fix only one did++ assert staged == [0, 1]+ assert _count(session) == 2diff --git a/tests/benchmarks/web_api/test_benchmark_service_coverage.py b/tests/benchmarks/web_api/test_benchmark_service_coverage.pyindex 4b20ef8712..d014ec65c2 100644--- a/tests/benchmarks/web_api/test_benchmark_service_coverage.py+++ b/tests/benchmarks/web_api/test_benchmark_service_coverage.py@@ -224,22 +224,29 @@ class TestHashGeneration:def test_query_hash_deterministic(self):svc = _make_service()- h1 = svc.generate_query_hash("What is AI?", "simpleqa")- h2 = svc.generate_query_hash("What is AI?", "simpleqa")+ h1 = svc.generate_query_hash("What is AI?", "simpleqa", "ex1")+ h2 = svc.generate_query_hash("What is AI?", "simpleqa", "ex1")assert h1 == h2def test_query_hash_strips_whitespace(self):svc = _make_service()- h1 = svc.generate_query_hash(" What is AI? ", "simpleqa")- h2 = svc.generate_query_hash("What is AI?", "simpleqa")+ h1 = svc.generate_query_hash(" What is AI? ", "simpleqa", "ex1")+ h2 = svc.generate_query_hash("What is AI?", "simpleqa", "ex1")assert h1 == h2def test_query_hash_case_insensitive_dataset(self):svc = _make_service()- h1 = svc.generate_query_hash("What is AI?", "SimpleQA")- h2 = svc.generate_query_hash("What is AI?", "simpleqa")+ h1 = svc.generate_query_hash("What is AI?", "SimpleQA", "ex1")+ h2 = svc.generate_query_hash("What is AI?", "simpleqa", "ex1")assert h1 == h2+ def test_query_hash_keys_on_example_id(self):+ # #4860: distinct examples, identical question -> distinct hashes.+ svc = _make_service()+ h1 = svc.generate_query_hash("What is AI?", "simpleqa", "ex1")+ h2 = svc.generate_query_hash("What is AI?", "simpleqa", "ex2")+ assert h1 != h2+# ============================================================# create_benchmark_run