~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Behavioral tests for text_cleaner module.
Tests the remove_surrogates function for handling encoding issues.
"""
class TestRemoveSurrogatesBasic:
"""Tests for basic remove_surrogates functionality."""
def test_returns_same_for_normal_text(self):
"""Returns same text for normal ASCII."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
text = "Hello World"
result = remove_surrogates(text)
assert result == text
def test_returns_same_for_utf8_text(self):
"""Returns same text for valid UTF-8."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
text = "Hello 世界 🌍"
result = remove_surrogates(text)
assert result == text
def test_handles_empty_string(self):
"""Handles empty string."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
result = remove_surrogates("")
assert result == ""
def test_handles_none(self):
"""Handles None input."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
result = remove_surrogates(None)
assert result is None
def test_returns_string(self):
"""Returns a string type."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
result = remove_surrogates("test")
assert isinstance(result, str)
class TestRemoveSurrogatesUnicode:
"""Tests for Unicode handling in remove_surrogates."""
def test_handles_chinese_characters(self):
"""Handles Chinese characters."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
text = "这是中文文本"
result = remove_surrogates(text)
assert result == text
def test_handles_japanese_characters(self):
"""Handles Japanese characters."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
text = "これは日本語です"
result = remove_surrogates(text)
assert result == text
def test_handles_korean_characters(self):
"""Handles Korean characters."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
text = "한국어 텍스트"
result = remove_surrogates(text)
assert result == text
def test_handles_arabic_characters(self):
"""Handles Arabic characters."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
text = "مرحبا بالعالم"
result = remove_surrogates(text)
assert result == text
def test_handles_emoji(self):
"""Handles emoji characters."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
text = "Hello 👋 World 🌍"
result = remove_surrogates(text)
assert result == text
def test_handles_mixed_scripts(self):
"""Handles mixed script text."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
text = "English, 中文, 日本語, 한국어"
result = remove_surrogates(text)
assert result == text
def test_handles_special_unicode_symbols(self):
"""Handles special Unicode symbols."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
text = "€ £ ¥ © ® ™ § ¶"
result = remove_surrogates(text)
assert result == text
class TestRemoveSurrogatesSurrogateHandling:
"""Tests for actual surrogate character handling."""
def test_removes_lone_high_surrogate(self):
"""Removes lone high surrogate character."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
# \ud800 is a high surrogate
text = "Hello\ud800World"
result = remove_surrogates(text)
# Should not contain the surrogate, and should be valid UTF-8
assert result.encode("utf-8") is not None
assert "\ud800" not in result
def test_removes_lone_low_surrogate(self):
"""Removes lone low surrogate character."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
# \udc00 is a low surrogate
text = "Hello\udc00World"
result = remove_surrogates(text)
# Should not contain the surrogate
assert result.encode("utf-8") is not None
assert "\udc00" not in result
def test_removes_multiple_surrogates(self):
"""Removes multiple surrogate characters."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
text = "\ud800text\ud801more\udc00text"
result = remove_surrogates(text)
assert result.encode("utf-8") is not None
def test_preserves_text_around_surrogates(self):
"""Preserves text around surrogates."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
text = "before\ud800after"
result = remove_surrogates(text)
assert "before" in result
assert "after" in result
class TestRemoveSurrogatesEdgeCases:
"""Tests for edge cases in remove_surrogates."""
def test_handles_very_long_text(self):
"""Handles very long text."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
text = "a" * 100000
result = remove_surrogates(text)
assert len(result) == 100000
def test_handles_newlines(self):
"""Handles text with newlines."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
text = "Line 1\nLine 2\nLine 3"
result = remove_surrogates(text)
assert result == text
def test_handles_tabs(self):
"""Handles text with tabs."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
text = "Col1\tCol2\tCol3"
result = remove_surrogates(text)
assert result == text
def test_handles_carriage_returns(self):
"""Handles text with carriage returns."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
text = "Line 1\r\nLine 2"
result = remove_surrogates(text)
assert result == text
def test_handles_null_characters(self):
"""Handles text with null characters."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
text = "before\x00after"
result = remove_surrogates(text)
# Null character should be preserved as it's valid UTF-8
assert result == text
def test_handles_whitespace_only(self):
"""Handles whitespace-only text."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
text = " \t\n "
result = remove_surrogates(text)
assert result == text
class TestRemoveSurrogatesBoundaryCodepoints:
"""Tests for boundary Unicode codepoints."""
def test_handles_bmp_boundary(self):
"""Handles characters at BMP boundary."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
# U+FFFF is last character in BMP
text = "test\ufffftest"
result = remove_surrogates(text)
assert isinstance(result, str)
# Should be encodable as UTF-8
result.encode("utf-8")
def test_handles_supplementary_planes(self):
"""Handles characters from supplementary planes."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
# 𐀀 is U+10000, first supplementary character
text = "test\U00010000test"
result = remove_surrogates(text)
assert result == text
def test_handles_high_codepoints(self):
"""Handles high Unicode codepoints."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
# U+1F600 is 😀
text = "test\U0001f600test"
result = remove_surrogates(text)
assert result == text
class TestRemoveSurrogatesRealWorldScenarios:
"""Tests for real-world scenarios that produce surrogates."""
def test_handles_pdf_extraction_style_text(self):
"""Handles text that might come from PDF extraction."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
# PDF extraction sometimes produces ligatures and special chars
text = "final flow ffect"
result = remove_surrogates(text)
assert result == text
def test_handles_mathematical_symbols(self):
"""Handles mathematical symbols."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
text = "∑ ∏ ∫ √ ∞ ≈ ≠ ≤ ≥"
result = remove_surrogates(text)
assert result == text
def test_handles_greek_letters(self):
"""Handles Greek letters (common in scientific text)."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
text = "α β γ δ ε ζ η θ"
result = remove_surrogates(text)
assert result == text
def test_result_is_utf8_encodable(self):
"""Result is always UTF-8 encodable."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
# Mix of normal text and surrogates
text = "normal\ud800text\udc00more"
result = remove_surrogates(text)
# Should not raise
encoded = result.encode("utf-8")
assert isinstance(encoded, bytes)
class TestRemoveSurrogatesConsistency:
"""Tests for consistency of remove_surrogates behavior."""
def test_idempotent(self):
"""Applying twice gives same result."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
text = "Hello\ud800World"
result1 = remove_surrogates(text)
result2 = remove_surrogates(result1)
assert result1 == result2
def test_preserves_length_for_clean_text(self):
"""Preserves length for text without surrogates."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
text = "Hello World"
result = remove_surrogates(text)
assert len(result) == len(text)
def test_deterministic(self):
"""Same input always produces same output."""
from local_deep_research.text_processing.text_cleaner import (
remove_surrogates,
)
text = "test\ud800text"
result1 = remove_surrogates(text)
result2 = remove_surrogates(text)
assert result1 == result2