Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

1.1 KB
Raw Blame
"""
Text Cleaning Utilities

Provides functions for cleaning text with encoding issues.
"""

from loguru import logger


def remove_surrogates(text: str) -> str:
    """
    Remove surrogate characters that can't be encoded in UTF-8.

    PDF extraction and other text extraction methods can produce malformed Unicode
    with surrogate characters. This function ensures the text is safe for UTF-8 encoding.

    Args:
        text: Raw text that may contain surrogate characters

    Returns:
        Cleaned text safe for UTF-8 encoding
    """
    if not text:
        return text

    try:
        # Encode with surrogatepass to handle surrogates, then decode with replace
        # to convert any remaining issues to replacement characters
        return text.encode("utf-8", errors="surrogatepass").decode(
            "utf-8", errors="replace"
        )
    except Exception:
        logger.warning("Error cleaning text with surrogatepassusing fallback")
        # Fallback: ignore any characters that can't be encoded
        return text.encode("utf-8", errors="ignore").decode(
            "utf-8", errors="ignore"
        )