~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Tests for text_cleaner module."""
from local_deep_research.text_processing.text_cleaner import remove_surrogates
class TestRemoveSurrogates:
"""Tests for remove_surrogates function."""
def test_returns_none_for_none_input(self):
"""Should return None for None input."""
result = remove_surrogates(None)
assert result is None
def test_returns_empty_string_for_empty_input(self):
"""Should return empty string for empty input."""
result = remove_surrogates("")
assert result == ""
def test_returns_normal_text_unchanged(self):
"""Should return normal text unchanged."""
text = "Hello, World! This is normal text."
result = remove_surrogates(text)
assert result == text
def test_handles_unicode_text(self):
"""Should handle valid Unicode text."""
text = "Hello, World! Test text"
result = remove_surrogates(text)
assert result == text
def test_removes_surrogate_characters(self):
"""Should remove/replace surrogate characters."""
# Surrogate characters are in range U+D800 to U+DFFF
text_with_surrogate = "Hello\ud800World"
result = remove_surrogates(text_with_surrogate)
# Result should be encodable as UTF-8
result.encode("utf-8") # Should not raise
# Surrogate should be replaced
assert "\ud800" not in result
def test_handles_mixed_valid_and_invalid(self):
"""Should handle text with both valid and invalid characters."""
text = "Valid text\ud800more valid"
result = remove_surrogates(text)
# Should preserve valid parts
assert "Valid text" in result
assert "more valid" in result
# Should be encodable
result.encode("utf-8")
def test_preserves_newlines(self):
"""Should preserve newlines and whitespace."""
text = "Line 1\nLine 2\r\nLine 3\tTabbed"
result = remove_surrogates(text)
assert "\n" in result
assert "\t" in result
def test_handles_special_characters(self):
"""Should handle special characters correctly."""
text = "<html> test'\"quote</html>"
result = remove_surrogates(text)
assert result == text
def test_handles_long_text(self):
"""Should handle long text efficiently."""
text = "A" * 10000
result = remove_surrogates(text)
assert len(result) == 10000
def test_handles_consecutive_surrogates(self):
"""Should handle multiple consecutive surrogate characters."""
text = "Start\ud800\ud801\ud802End"
result = remove_surrogates(text)
# Should be encodable
result.encode("utf-8")
assert "Start" in result
assert "End" in result
def test_exception_handler_fallback(self):
"""Should use ignore-based fallback when encode raises an exception.
Uses a str subclass whose encode method raises on surrogatepass,
forcing the except branch (lines 32-38) to execute.
"""
class BadString(str):
_call_count = 0
def encode(self, encoding="utf-8", errors="strict"):
BadString._call_count += 1
if errors == "surrogatepass":
raise RuntimeError("Simulated encode failure")
return super().encode(encoding, errors=errors)
BadString._call_count = 0
text = BadString("hello world")
result = remove_surrogates(text)
assert isinstance(result, str)
assert "hello" in result
# surrogatepass was attempted (triggering exception),
# then ignore was used as fallback
assert BadString._call_count >= 2