Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

test: delete shadow tests/advanced_search_system/evidence/test_evidence_evaluator.py (#4272)

LearningCircuit1eda1809eb92
diff --git a/tests/advanced_search_system/evidence/test_evidence_evaluator.py b/tests/advanced_search_system/evidence/test_evidence_evaluator.py
deleted file mode 100644
index 571826ce96..0000000000
--- a/tests/advanced_search_system/evidence/test_evidence_evaluator.py
+++ /dev/null
@@ -1,442 +0,0 @@
-"""
-Tests for EvidenceEvaluator - Evidence quality and relevance evaluation.
-
-Tests cover:
-- Evaluator initialization
-- Evidence extraction from search results
-- Evidence type parsing
-- Confidence scoring
-- Source reliability weights
-- Match quality assessment
-"""
-
-
-class TestEvidenceEvaluatorInitialization:
- """Tests for EvidenceEvaluator initialization."""
-
- def test_source_reliability_weights(self):
- """Should have correct source reliability weights."""
- source_reliability = {
- "official": 1.0,
- "research": 0.95,
- "news": 0.8,
- "community": 0.6,
- "inference": 0.5,
- "speculation": 0.3,
- }
-
- assert source_reliability["official"] == 1.0
- assert source_reliability["research"] == 0.95
- assert source_reliability["speculation"] == 0.3
-
- def test_reliability_ordering(self):
- """Source reliability should be properly ordered."""
- source_reliability = {
- "official": 1.0,
- "research": 0.95,
- "news": 0.8,
- "community": 0.6,
- "inference": 0.5,
- "speculation": 0.3,
- }
-
- sorted_sources = sorted(
- source_reliability.items(), key=lambda x: x[1], reverse=True
- )
-
- assert sorted_sources[0][0] == "official"
- assert sorted_sources[-1][0] == "speculation"
-
-
-class TestEvidenceExtraction:
- """Tests for evidence extraction."""
-
- def test_extract_evidence_prompt_structure(self):
- """Prompt should include all required fields."""
- candidate = "TestCandidate"
- constraint_description = "Must be over 18"
- constraint_type = "AGE"
- constraint_value = "18+"
- search_result = "Some search result text..."
-
- prompt = f"""
-Extract evidence regarding whether "{candidate}" satisfies this constraint:
-
-Constraint: {constraint_description}
-Constraint Type: {constraint_type}
-Required Value: {constraint_value}
-
-Search Results:
-{search_result[:3000]}
-"""
-
- assert candidate in prompt
- assert constraint_description in prompt
- assert "CLAIM:" not in prompt # Part of expected response format
-
- def test_parse_evidence_response_claim(self):
- """Should parse claim from response."""
- response = """CLAIM: The candidate meets the age requirement
-TYPE: direct_statement
-SOURCE: official records
-CONFIDENCE: 0.9
-REASONING: Based on documented evidence
-QUOTE: "Age verified as 25"""
-
- parsed = {}
- for line in response.strip().split("\n"):
- if ":" in line:
- key, value = line.split(":", 1)
- key = key.strip().lower()
- value = value.strip()
- if key in [
- "claim",
- "type",
- "source",
- "confidence",
- "reasoning",
- "quote",
- ]:
- parsed[key] = value
-
- assert parsed["claim"] == "The candidate meets the age requirement"
-
- def test_parse_evidence_response_confidence(self):
- """Should parse confidence as float."""
- import re
-
- response = "CONFIDENCE: 0.85"
-
- for line in response.strip().split("\n"):
- if ":" in line:
- key, value = line.split(":", 1)
- if key.strip().lower() == "confidence":
- match = re.search(r"(\d*\.?\d+)", value)
- if match:
- confidence = float(match.group(1))
-
- assert confidence == 0.85
-
- def test_parse_confidence_with_text(self):
- """Should extract confidence from text with numbers."""
- import re
-
- value = "0.75 (high confidence based on multiple sources)"
- match = re.search(r"(\d*\.?\d+)", value)
- if match:
- confidence = float(match.group(1))
-
- assert confidence == 0.75
-
-
-class TestEvidenceTypeParsing:
- """Tests for evidence type parsing."""
-
- def test_parse_direct_statement(self):
- """Should parse direct_statement type."""
- type_map = {
- "direct_statement": "DIRECT_STATEMENT",
- "official_record": "OFFICIAL_RECORD",
- "speculation": "SPECULATION",
- }
-
- type_str = "direct_statement"
- parsed_type = type_map.get(type_str.lower(), "SPECULATION")
-
- assert parsed_type == "DIRECT_STATEMENT"
-
- def test_parse_official_record(self):
- """Should parse official_record type."""
- type_map = {
- "direct_statement": "DIRECT_STATEMENT",
- "official_record": "OFFICIAL_RECORD",
- }
-
- type_str = "official_record"
- parsed_type = type_map.get(type_str.lower(), "SPECULATION")
-
- assert parsed_type == "OFFICIAL_RECORD"
-
- def test_parse_unknown_type_defaults_to_speculation(self):
- """Unknown types should default to speculation."""
- type_map = {
- "direct_statement": "DIRECT_STATEMENT",
- "official_record": "OFFICIAL_RECORD",
- }
-
- type_str = "unknown_type"
- parsed_type = type_map.get(type_str.lower(), "SPECULATION")
-
- assert parsed_type == "SPECULATION"
-
- def test_all_evidence_types_mapped(self):
- """All evidence types should be mapped."""
- type_map = {
- "direct_statement": "DIRECT_STATEMENT",
- "official_record": "OFFICIAL_RECORD",
- "research_finding": "RESEARCH_FINDING",
- "news_report": "NEWS_REPORT",
- "statistical_data": "STATISTICAL_DATA",
- "inference": "INFERENCE",
- "correlation": "CORRELATION",
- "speculation": "SPECULATION",
- }
-
- assert len(type_map) == 8
- assert "direct_statement" in type_map
- assert "speculation" in type_map
-
-
-class TestConfidenceScoring:
- """Tests for confidence scoring."""
-
- def test_confidence_clamped_to_range(self):
- """Confidence should be clamped between 0 and 1."""
- confidence_str = "1.5"
-
- try:
- confidence = float(confidence_str)
- confidence = max(0.0, min(1.0, confidence))
- except ValueError:
- confidence = 0.5
-
- assert confidence == 1.0
-
- def test_confidence_negative_clamped(self):
- """Negative confidence should be clamped to 0."""
- confidence_str = "-0.5"
-
- try:
- confidence = float(confidence_str)
- confidence = max(0.0, min(1.0, confidence))
- except ValueError:
- confidence = 0.5
-
- assert confidence == 0.0
-
- def test_confidence_parse_failure_default(self):
- """Parse failure should default to 0.5."""
- confidence_str = "high"
-
- try:
- confidence = float(confidence_str)
- except ValueError:
- confidence = 0.5
-
- assert confidence == 0.5
-
- def test_confidence_valid_range(self):
- """Valid confidence should be preserved."""
- confidence_str = "0.75"
-
- try:
- confidence = float(confidence_str)
- confidence = max(0.0, min(1.0, confidence))
- except ValueError:
- confidence = 0.5
-
- assert confidence == 0.75
-
-
-class TestMatchQualityAssessment:
- """Tests for match quality assessment."""
-
- def test_exact_match_full_quality(self):
- """Exact match should return 1.0 quality."""
- constraint_value = "blue color"
- claim = "The item has a blue color finish"
-
- if constraint_value.lower() in claim.lower():
- quality = 1.0
- elif any(
- word in claim.lower() for word in constraint_value.lower().split()
- ):
- quality = 0.8
- else:
- quality = 0.6
-
- assert quality == 1.0
-
- def test_partial_match_reduced_quality(self):
- """Partial word match should return 0.8 quality."""
- constraint_value = "blue color"
- claim = "The item has a blue tint"
-
- if constraint_value.lower() in claim.lower():
- quality = 1.0
- elif any(
- word in claim.lower() for word in constraint_value.lower().split()
- ):
- quality = 0.8
- else:
- quality = 0.6
-
- assert quality == 0.8
-
- def test_no_match_minimum_quality(self):
- """No match should return 0.6 quality."""
- constraint_value = "blue color"
- claim = "The item is red"
-
- if constraint_value.lower() in claim.lower():
- quality = 1.0
- elif any(
- word in claim.lower() for word in constraint_value.lower().split()
- ):
- quality = 0.8
- else:
- quality = 0.6
-
- assert quality == 0.6
-
- def test_confidence_adjusted_by_quality(self):
- """Confidence should be adjusted by match quality."""
- base_confidence = 0.9
- match_quality = 0.8
-
- adjusted_confidence = base_confidence * match_quality
-
- assert abs(adjusted_confidence - 0.72) < 0.001
-
-
-class TestEvidenceObject:
- """Tests for Evidence object structure."""
-
- def test_evidence_structure(self):
- """Evidence should have all required fields."""
- evidence = {
- "claim": "Test claim",
- "type": "DIRECT_STATEMENT",
- "source": "Test source",
- "confidence": 0.85,
- "reasoning": "Test reasoning",
- "raw_text": "Test quote",
- "metadata": {
- "candidate": "TestCandidate",
- "constraint_id": "c1",
- "constraint_type": "VALUE_MATCH",
- },
- }
-
- assert "claim" in evidence
- assert "type" in evidence
- assert "source" in evidence
- assert "confidence" in evidence
- assert "metadata" in evidence
-
- def test_evidence_metadata_structure(self):
- """Evidence metadata should include context."""
- metadata = {
- "candidate": "TestCandidate",
- "constraint_id": "constraint_1",
- "constraint_type": "AGE_REQUIREMENT",
- }
-
- assert metadata["candidate"] == "TestCandidate"
- assert metadata["constraint_id"] == "constraint_1"
-
- def test_evidence_default_claim(self):
- """Missing claim should default to 'No clear claim'."""
- parsed = {}
- claim = parsed.get("claim", "No clear claim")
-
- assert claim == "No clear claim"
-
- def test_evidence_default_source(self):
- """Missing source should default to 'Unknown'."""
- parsed = {}
- source = parsed.get("source", "Unknown")
-
- assert source == "Unknown"
-
-
-class TestSearchResultTruncation:
- """Tests for search result truncation."""
-
- def test_search_result_truncated_to_3000(self):
- """Search results should be truncated to 3000 chars."""
- search_result = "x" * 5000
- truncated = search_result[:3000]
-
- assert len(truncated) == 3000
-
- def test_short_search_result_not_truncated(self):
- """Short search results should not be truncated."""
- search_result = "x" * 1000
- truncated = search_result[:3000]
-
- assert len(truncated) == 1000
-
-
-class TestResponseParsing:
- """Tests for LLM response parsing."""
-
- def test_parse_multiline_response(self):
- """Should parse multiline response correctly."""
- response = """CLAIM: The candidate is 25 years old
-TYPE: official_record
-SOURCE: Government database
-CONFIDENCE: 0.95
-REASONING: Age verified through official channels
-QUOTE: "Date of birth: January 1, 1999"""
-
- parsed = {}
- for line in response.strip().split("\n"):
- if ":" in line:
- key, value = line.split(":", 1)
- key = key.strip().lower()
- value = value.strip()
- if key in [
- "claim",
- "type",
- "source",
- "confidence",
- "reasoning",
- "quote",
- ]:
- parsed[key] = value
-
- assert len(parsed) == 6
- assert parsed["type"] == "official_record"
- assert parsed["source"] == "Government database"
-
- def test_parse_response_with_extra_lines(self):
- """Should ignore lines without key:value format."""
- response = """Here is the evidence:
-
-CLAIM: Test claim
-TYPE: direct_statement
-
-Additional notes follow."""
-
- parsed = {}
- for line in response.strip().split("\n"):
- if ":" in line:
- key, value = line.split(":", 1)
- key = key.strip().lower()
- value = value.strip()
- if key in ["claim", "type"]:
- parsed[key] = value
-
- assert "claim" in parsed
- assert "type" in parsed
- assert len(parsed) == 2
-
- def test_parse_response_missing_fields(self):
- """Should handle missing fields gracefully."""
- response = """CLAIM: Partial evidence
-CONFIDENCE: 0.7"""
-
- parsed = {}
- for line in response.strip().split("\n"):
- if ":" in line:
- key, value = line.split(":", 1)
- key = key.strip().lower()
- value = value.strip()
- if key in ["claim", "type", "source", "confidence"]:
- parsed[key] = value
-
- assert "claim" in parsed
- assert "confidence" in parsed
- assert "type" not in parsed
- assert "source" not in parsed