Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix: improve binary content detection and null byte handling (#210)
bot777454e7279a71b
diff --git a/openviking/parse/parsers/upload_utils.py b/openviking/parse/parsers/upload_utils.pyindex b675cd548..39f73744f 100644--- a/openviking/parse/parsers/upload_utils.py+++ b/openviking/parse/parsers/upload_utils.py@@ -62,10 +62,24 @@ def detect_and_convert_encoding(content: bytes, file_path: Union[str, Path] = ""return contenttry:+ # Check for potential binary content (null bytes in first 8KB)+ # Binary files often contain null bytes which can cause issues+ sample_size = min(8192, len(content))+ if b'\x00' in content[:sample_size]:+ null_count = content[:sample_size].count(b'\x00')+ # If more than 5% null bytes in sample, likely binary - don't process+ if null_count / sample_size > 0.05:+ logger.debug(f"Detected binary content in {file_path} (null bytes: {null_count}), skipping encoding detection")+ return content+detected_encoding: Optional[str] = Nonefor encoding in TEXT_ENCODINGS:try:- content.decode(encoding)+ decoded = content.decode(encoding)+ # Additional validation: check for control characters that suggest binary+ control_chars = sum(1 for c in decoded[:1000] if ord(c) < 32 and c not in '\t\n\r')+ if control_chars / min(1000, len(decoded)) > 0.05: # More than 5% control chars+ continuedetected_encoding = encodingbreakexcept UnicodeDecodeError:@@ -77,6 +91,10 @@ def detect_and_convert_encoding(content: bytes, file_path: Union[str, Path] = ""if detected_encoding not in UTF8_VARIANTS:decoded_content = content.decode(detected_encoding, errors="replace")+ # Remove null bytes from decoded content as they can cause issues downstream+ if '\x00' in decoded_content:+ decoded_content = decoded_content.replace('\x00', '')+ logger.debug(f"Removed null bytes from decoded content in {file_path}")content = decoded_content.encode("utf-8")logger.debug(f"Converted {file_path} from {detected_encoding} to UTF-8")