Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(benchmarks): include example_id in query hash so distinct examples persist (#5080)

Ehsan178eddc2bf2f
diff --git a/changelog.d/4860.bugfix.md b/changelog.d/4860.bugfix.md
new file mode 100644
index 0000000000..c2d884d176
--- /dev/null
+++ b/changelog.d/4860.bugfix.md
@@ -0,0 +1 @@
+Benchmark runs now persist one result row per example even when two examples have identical question text. Previously the per-query hash keyed only on the question and dataset, so distinct examples with the same question collided on the `uix_run_query` unique constraint and all but the first were dropped; the stored row count fell below the number of examples executed and disagreed with the in-memory accuracy. The example id is now part of the hash, so each example persists its own row while a genuine re-sync of the same example is still deduplicated.
diff --git a/src/local_deep_research/benchmarks/web_api/benchmark_service.py b/src/local_deep_research/benchmarks/web_api/benchmark_service.py
index b3aedc85a7..eff9aedf06 100644
--- a/src/local_deep_research/benchmarks/web_api/benchmark_service.py
+++ b/src/local_deep_research/benchmarks/web_api/benchmark_service.py
@@ -186,9 +186,20 @@ class BenchmarkService:
config_str.encode(), usedforsecurity=False
).hexdigest()[:8]
- def generate_query_hash(self, question: str, dataset_type: str) -> str:
- """Generate a hash for a query to enable deduplication."""
- query_content = f"{question.strip()}|{dataset_type.lower()}"
+ def generate_query_hash(
+ self, question: str, dataset_type: str, example_id: str
+ ) -> str:
+ """Generate a hash for a query to enable deduplication.
+
+ The example id is part of the key so two distinct examples that share
+ the same question text (and dataset) get distinct hashes and each
+ persist a row, instead of colliding on the ``uix_run_query`` unique
+ constraint and dropping one. Re-hashing the same example is stable, so
+ the sync dedup still collapses a genuine re-sync of that example.
+ """
+ query_content = (
+ f"{example_id}|{question.strip()}|{dataset_type.lower()}"
+ )
return hashlib.md5( # DevSkim: ignore DS126858
query_content.encode(), usedforsecurity=False
).hexdigest()
@@ -626,16 +637,18 @@ class BenchmarkService:
# All registered datasets expose "problem"/"answer"
question = example.get("problem", "")
correct_answer = example.get("answer", "")
+ example_id = example.get("id", f"example_{i}")
- # Generate query hash
+ # Generate query hash (keyed on the example so distinct
+ # examples with identical question text don't collide)
query_hash = self.generate_query_hash(
- question, dataset_name
+ question, dataset_name, example_id
)
tasks.append(
{
"benchmark_run_id": benchmark_run_id,
- "example_id": example.get("id", f"example_{i}"),
+ "example_id": example_id,
"dataset_type": dataset_name,
"question": question,
"correct_answer": correct_answer,
diff --git a/tests/benchmarks/test_benchmark_service.py b/tests/benchmarks/test_benchmark_service.py
index d5750a8e2c..6b3105a68c 100644
--- a/tests/benchmarks/test_benchmark_service.py
+++ b/tests/benchmarks/test_benchmark_service.py
@@ -377,13 +377,39 @@ class TestBenchmarkServiceQueryHash:
mock_socket.return_value = Mock()
service = BenchmarkService()
- hash1 = service.generate_query_hash("What is AI?", "simpleqa")
- hash2 = service.generate_query_hash("What is AI?", "simpleqa")
+ hash1 = service.generate_query_hash(
+ "What is AI?", "simpleqa", "ex1"
+ )
+ hash2 = service.generate_query_hash(
+ "What is AI?", "simpleqa", "ex1"
+ )
- # Same question and dataset should produce same hash
+ # Same question, dataset and example should produce same hash
assert hash1 == hash2
assert len(hash1) == 32 # MD5 hex digest
+ def test_different_example_ids_different_hashes(self):
+ """#4860: distinct examples with identical question text must not
+ collide, otherwise the sync dedup drops all but the first row."""
+ from local_deep_research.benchmarks.web_api.benchmark_service import (
+ BenchmarkService,
+ )
+
+ with patch(
+ "local_deep_research.benchmarks.web_api.benchmark_service.SocketIOService"
+ ) as mock_socket:
+ mock_socket.return_value = Mock()
+ service = BenchmarkService()
+
+ hash1 = service.generate_query_hash(
+ "Same question", "simpleqa", "ex1"
+ )
+ hash2 = service.generate_query_hash(
+ "Same question", "simpleqa", "ex2"
+ )
+
+ assert hash1 != hash2
+
def test_different_questions_different_hashes(self):
"""Test that different questions produce different hashes."""
from local_deep_research.benchmarks.web_api.benchmark_service import (
@@ -396,8 +422,8 @@ class TestBenchmarkServiceQueryHash:
mock_socket.return_value = Mock()
service = BenchmarkService()
- hash1 = service.generate_query_hash("Question 1", "simpleqa")
- hash2 = service.generate_query_hash("Question 2", "simpleqa")
+ hash1 = service.generate_query_hash("Question 1", "simpleqa", "ex1")
+ hash2 = service.generate_query_hash("Question 2", "simpleqa", "ex1")
assert hash1 != hash2
@@ -413,8 +439,12 @@ class TestBenchmarkServiceQueryHash:
mock_socket.return_value = Mock()
service = BenchmarkService()
- hash1 = service.generate_query_hash("Same question", "simpleqa")
- hash2 = service.generate_query_hash("Same question", "browsecomp")
+ hash1 = service.generate_query_hash(
+ "Same question", "simpleqa", "ex1"
+ )
+ hash2 = service.generate_query_hash(
+ "Same question", "browsecomp", "ex1"
+ )
assert hash1 != hash2
@@ -430,8 +460,10 @@ class TestBenchmarkServiceQueryHash:
mock_socket.return_value = Mock()
service = BenchmarkService()
- hash1 = service.generate_query_hash(" Question ", "simpleqa")
- hash2 = service.generate_query_hash("Question", "simpleqa")
+ hash1 = service.generate_query_hash(
+ " Question ", "simpleqa", "ex1"
+ )
+ hash2 = service.generate_query_hash("Question", "simpleqa", "ex1")
assert hash1 == hash2
diff --git a/tests/benchmarks/web_api/test_benchmark_result_dedup.py b/tests/benchmarks/web_api/test_benchmark_result_dedup.py
index f4dd8ef080..7ce70e807d 100644
--- a/tests/benchmarks/web_api/test_benchmark_result_dedup.py
+++ b/tests/benchmarks/web_api/test_benchmark_result_dedup.py
@@ -431,3 +431,41 @@ def test_callers_do_not_mark_saved_on_commit_failure():
with patch(target, _mock_session_cm(mock_session)):
svc._sync_results_to_database(2)
assert "saved_indices" not in run2, run2.get("saved_indices")
+
+
+# ---------------------------------------------------------------------------
+# #4860: the query_hash must key on the example, not just the question text
+# ---------------------------------------------------------------------------
+
+
+def test_distinct_examples_same_question_persist_separately():
+ """#4860: two DISTINCT examples that share identical question text (and
+ dataset) must each persist a row.
+
+ ``generate_query_hash`` used to key only on ``question`` + ``dataset``, so
+ two different examples with the same question produced the SAME hash and
+ the sync dedup (``uix_run_query``) dropped the second, so the stored row
+ count fell below the number of examples executed. Driven end-to-end
+ through the real ``_create_task_queue`` (which computes the hash) and
+ ``_persist_unsaved_results`` (which enforces the constraint)."""
+ svc = BenchmarkService(socket_service=MagicMock())
+ session = _make_session()
+
+ dataset = [
+ {"id": "ex_A", "problem": "Capital of France?", "answer": "Paris"},
+ {"id": "ex_B", "problem": "Capital of France?", "answer": "Paris"},
+ ]
+ with patch(
+ "local_deep_research.benchmarks.web_api.benchmark_service.load_dataset",
+ return_value=dataset,
+ ):
+ tasks = svc._create_task_queue({"simpleqa": {"count": 2}}, 1)
+
+ # Distinct examples -> distinct hashes, even with identical question text.
+ assert len({t["query_hash"] for t in tasks}) == 2
+
+ staged = svc._persist_unsaved_results(session, 1, {"results": tasks})
+ session.commit() # both rows land; before the fix only one did
+
+ assert staged == [0, 1]
+ assert _count(session) == 2
diff --git a/tests/benchmarks/web_api/test_benchmark_service_coverage.py b/tests/benchmarks/web_api/test_benchmark_service_coverage.py
index 4b20ef8712..d014ec65c2 100644
--- a/tests/benchmarks/web_api/test_benchmark_service_coverage.py
+++ b/tests/benchmarks/web_api/test_benchmark_service_coverage.py
@@ -224,22 +224,29 @@ class TestHashGeneration:
def test_query_hash_deterministic(self):
svc = _make_service()
- h1 = svc.generate_query_hash("What is AI?", "simpleqa")
- h2 = svc.generate_query_hash("What is AI?", "simpleqa")
+ h1 = svc.generate_query_hash("What is AI?", "simpleqa", "ex1")
+ h2 = svc.generate_query_hash("What is AI?", "simpleqa", "ex1")
assert h1 == h2
def test_query_hash_strips_whitespace(self):
svc = _make_service()
- h1 = svc.generate_query_hash(" What is AI? ", "simpleqa")
- h2 = svc.generate_query_hash("What is AI?", "simpleqa")
+ h1 = svc.generate_query_hash(" What is AI? ", "simpleqa", "ex1")
+ h2 = svc.generate_query_hash("What is AI?", "simpleqa", "ex1")
assert h1 == h2
def test_query_hash_case_insensitive_dataset(self):
svc = _make_service()
- h1 = svc.generate_query_hash("What is AI?", "SimpleQA")
- h2 = svc.generate_query_hash("What is AI?", "simpleqa")
+ h1 = svc.generate_query_hash("What is AI?", "SimpleQA", "ex1")
+ h2 = svc.generate_query_hash("What is AI?", "simpleqa", "ex1")
assert h1 == h2
+ def test_query_hash_keys_on_example_id(self):
+ # #4860: distinct examples, identical question -> distinct hashes.
+ svc = _make_service()
+ h1 = svc.generate_query_hash("What is AI?", "simpleqa", "ex1")
+ h2 = svc.generate_query_hash("What is AI?", "simpleqa", "ex2")
+ assert h1 != h2
+
# ============================================================
# create_benchmark_run