~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
test: delete shadow tests/advanced_search_system/evidence/test_evidence_evaluator.py (#4272)
LearningCircuit1eda1809eb92
diff --git a/tests/advanced_search_system/evidence/test_evidence_evaluator.py b/tests/advanced_search_system/evidence/test_evidence_evaluator.pydeleted file mode 100644index 571826ce96..0000000000--- a/tests/advanced_search_system/evidence/test_evidence_evaluator.py+++ /dev/null@@ -1,442 +0,0 @@-"""-Tests for EvidenceEvaluator - Evidence quality and relevance evaluation.--Tests cover:-- Evaluator initialization-- Evidence extraction from search results-- Evidence type parsing-- Confidence scoring-- Source reliability weights-- Match quality assessment-"""---class TestEvidenceEvaluatorInitialization:- """Tests for EvidenceEvaluator initialization."""-- def test_source_reliability_weights(self):- """Should have correct source reliability weights."""- source_reliability = {- "official": 1.0,- "research": 0.95,- "news": 0.8,- "community": 0.6,- "inference": 0.5,- "speculation": 0.3,- }-- assert source_reliability["official"] == 1.0- assert source_reliability["research"] == 0.95- assert source_reliability["speculation"] == 0.3-- def test_reliability_ordering(self):- """Source reliability should be properly ordered."""- source_reliability = {- "official": 1.0,- "research": 0.95,- "news": 0.8,- "community": 0.6,- "inference": 0.5,- "speculation": 0.3,- }-- sorted_sources = sorted(- source_reliability.items(), key=lambda x: x[1], reverse=True- )-- assert sorted_sources[0][0] == "official"- assert sorted_sources[-1][0] == "speculation"---class TestEvidenceExtraction:- """Tests for evidence extraction."""-- def test_extract_evidence_prompt_structure(self):- """Prompt should include all required fields."""- candidate = "TestCandidate"- constraint_description = "Must be over 18"- constraint_type = "AGE"- constraint_value = "18+"- search_result = "Some search result text..."-- prompt = f"""-Extract evidence regarding whether "{candidate}" satisfies this constraint:--Constraint: {constraint_description}-Constraint Type: {constraint_type}-Required Value: {constraint_value}--Search Results:-{search_result[:3000]}-"""-- assert candidate in prompt- assert constraint_description in prompt- assert "CLAIM:" not in prompt # Part of expected response format-- def test_parse_evidence_response_claim(self):- """Should parse claim from response."""- response = """CLAIM: The candidate meets the age requirement-TYPE: direct_statement-SOURCE: official records-CONFIDENCE: 0.9-REASONING: Based on documented evidence-QUOTE: "Age verified as 25"""-- parsed = {}- for line in response.strip().split("\n"):- if ":" in line:- key, value = line.split(":", 1)- key = key.strip().lower()- value = value.strip()- if key in [- "claim",- "type",- "source",- "confidence",- "reasoning",- "quote",- ]:- parsed[key] = value-- assert parsed["claim"] == "The candidate meets the age requirement"-- def test_parse_evidence_response_confidence(self):- """Should parse confidence as float."""- import re-- response = "CONFIDENCE: 0.85"-- for line in response.strip().split("\n"):- if ":" in line:- key, value = line.split(":", 1)- if key.strip().lower() == "confidence":- match = re.search(r"(\d*\.?\d+)", value)- if match:- confidence = float(match.group(1))-- assert confidence == 0.85-- def test_parse_confidence_with_text(self):- """Should extract confidence from text with numbers."""- import re-- value = "0.75 (high confidence based on multiple sources)"- match = re.search(r"(\d*\.?\d+)", value)- if match:- confidence = float(match.group(1))-- assert confidence == 0.75---class TestEvidenceTypeParsing:- """Tests for evidence type parsing."""-- def test_parse_direct_statement(self):- """Should parse direct_statement type."""- type_map = {- "direct_statement": "DIRECT_STATEMENT",- "official_record": "OFFICIAL_RECORD",- "speculation": "SPECULATION",- }-- type_str = "direct_statement"- parsed_type = type_map.get(type_str.lower(), "SPECULATION")-- assert parsed_type == "DIRECT_STATEMENT"-- def test_parse_official_record(self):- """Should parse official_record type."""- type_map = {- "direct_statement": "DIRECT_STATEMENT",- "official_record": "OFFICIAL_RECORD",- }-- type_str = "official_record"- parsed_type = type_map.get(type_str.lower(), "SPECULATION")-- assert parsed_type == "OFFICIAL_RECORD"-- def test_parse_unknown_type_defaults_to_speculation(self):- """Unknown types should default to speculation."""- type_map = {- "direct_statement": "DIRECT_STATEMENT",- "official_record": "OFFICIAL_RECORD",- }-- type_str = "unknown_type"- parsed_type = type_map.get(type_str.lower(), "SPECULATION")-- assert parsed_type == "SPECULATION"-- def test_all_evidence_types_mapped(self):- """All evidence types should be mapped."""- type_map = {- "direct_statement": "DIRECT_STATEMENT",- "official_record": "OFFICIAL_RECORD",- "research_finding": "RESEARCH_FINDING",- "news_report": "NEWS_REPORT",- "statistical_data": "STATISTICAL_DATA",- "inference": "INFERENCE",- "correlation": "CORRELATION",- "speculation": "SPECULATION",- }-- assert len(type_map) == 8- assert "direct_statement" in type_map- assert "speculation" in type_map---class TestConfidenceScoring:- """Tests for confidence scoring."""-- def test_confidence_clamped_to_range(self):- """Confidence should be clamped between 0 and 1."""- confidence_str = "1.5"-- try:- confidence = float(confidence_str)- confidence = max(0.0, min(1.0, confidence))- except ValueError:- confidence = 0.5-- assert confidence == 1.0-- def test_confidence_negative_clamped(self):- """Negative confidence should be clamped to 0."""- confidence_str = "-0.5"-- try:- confidence = float(confidence_str)- confidence = max(0.0, min(1.0, confidence))- except ValueError:- confidence = 0.5-- assert confidence == 0.0-- def test_confidence_parse_failure_default(self):- """Parse failure should default to 0.5."""- confidence_str = "high"-- try:- confidence = float(confidence_str)- except ValueError:- confidence = 0.5-- assert confidence == 0.5-- def test_confidence_valid_range(self):- """Valid confidence should be preserved."""- confidence_str = "0.75"-- try:- confidence = float(confidence_str)- confidence = max(0.0, min(1.0, confidence))- except ValueError:- confidence = 0.5-- assert confidence == 0.75---class TestMatchQualityAssessment:- """Tests for match quality assessment."""-- def test_exact_match_full_quality(self):- """Exact match should return 1.0 quality."""- constraint_value = "blue color"- claim = "The item has a blue color finish"-- if constraint_value.lower() in claim.lower():- quality = 1.0- elif any(- word in claim.lower() for word in constraint_value.lower().split()- ):- quality = 0.8- else:- quality = 0.6-- assert quality == 1.0-- def test_partial_match_reduced_quality(self):- """Partial word match should return 0.8 quality."""- constraint_value = "blue color"- claim = "The item has a blue tint"-- if constraint_value.lower() in claim.lower():- quality = 1.0- elif any(- word in claim.lower() for word in constraint_value.lower().split()- ):- quality = 0.8- else:- quality = 0.6-- assert quality == 0.8-- def test_no_match_minimum_quality(self):- """No match should return 0.6 quality."""- constraint_value = "blue color"- claim = "The item is red"-- if constraint_value.lower() in claim.lower():- quality = 1.0- elif any(- word in claim.lower() for word in constraint_value.lower().split()- ):- quality = 0.8- else:- quality = 0.6-- assert quality == 0.6-- def test_confidence_adjusted_by_quality(self):- """Confidence should be adjusted by match quality."""- base_confidence = 0.9- match_quality = 0.8-- adjusted_confidence = base_confidence * match_quality-- assert abs(adjusted_confidence - 0.72) < 0.001---class TestEvidenceObject:- """Tests for Evidence object structure."""-- def test_evidence_structure(self):- """Evidence should have all required fields."""- evidence = {- "claim": "Test claim",- "type": "DIRECT_STATEMENT",- "source": "Test source",- "confidence": 0.85,- "reasoning": "Test reasoning",- "raw_text": "Test quote",- "metadata": {- "candidate": "TestCandidate",- "constraint_id": "c1",- "constraint_type": "VALUE_MATCH",- },- }-- assert "claim" in evidence- assert "type" in evidence- assert "source" in evidence- assert "confidence" in evidence- assert "metadata" in evidence-- def test_evidence_metadata_structure(self):- """Evidence metadata should include context."""- metadata = {- "candidate": "TestCandidate",- "constraint_id": "constraint_1",- "constraint_type": "AGE_REQUIREMENT",- }-- assert metadata["candidate"] == "TestCandidate"- assert metadata["constraint_id"] == "constraint_1"-- def test_evidence_default_claim(self):- """Missing claim should default to 'No clear claim'."""- parsed = {}- claim = parsed.get("claim", "No clear claim")-- assert claim == "No clear claim"-- def test_evidence_default_source(self):- """Missing source should default to 'Unknown'."""- parsed = {}- source = parsed.get("source", "Unknown")-- assert source == "Unknown"---class TestSearchResultTruncation:- """Tests for search result truncation."""-- def test_search_result_truncated_to_3000(self):- """Search results should be truncated to 3000 chars."""- search_result = "x" * 5000- truncated = search_result[:3000]-- assert len(truncated) == 3000-- def test_short_search_result_not_truncated(self):- """Short search results should not be truncated."""- search_result = "x" * 1000- truncated = search_result[:3000]-- assert len(truncated) == 1000---class TestResponseParsing:- """Tests for LLM response parsing."""-- def test_parse_multiline_response(self):- """Should parse multiline response correctly."""- response = """CLAIM: The candidate is 25 years old-TYPE: official_record-SOURCE: Government database-CONFIDENCE: 0.95-REASONING: Age verified through official channels-QUOTE: "Date of birth: January 1, 1999"""-- parsed = {}- for line in response.strip().split("\n"):- if ":" in line:- key, value = line.split(":", 1)- key = key.strip().lower()- value = value.strip()- if key in [- "claim",- "type",- "source",- "confidence",- "reasoning",- "quote",- ]:- parsed[key] = value-- assert len(parsed) == 6- assert parsed["type"] == "official_record"- assert parsed["source"] == "Government database"-- def test_parse_response_with_extra_lines(self):- """Should ignore lines without key:value format."""- response = """Here is the evidence:--CLAIM: Test claim-TYPE: direct_statement--Additional notes follow."""-- parsed = {}- for line in response.strip().split("\n"):- if ":" in line:- key, value = line.split(":", 1)- key = key.strip().lower()- value = value.strip()- if key in ["claim", "type"]:- parsed[key] = value-- assert "claim" in parsed- assert "type" in parsed- assert len(parsed) == 2-- def test_parse_response_missing_fields(self):- """Should handle missing fields gracefully."""- response = """CLAIM: Partial evidence-CONFIDENCE: 0.7"""-- parsed = {}- for line in response.strip().split("\n"):- if ":" in line:- key, value = line.split(":", 1)- key = key.strip().lower()- value = value.strip()- if key in ["claim", "type", "source", "confidence"]:- parsed[key] = value-- assert "claim" in parsed- assert "confidence" in parsed- assert "type" not in parsed- assert "source" not in parsed