Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(language): add threshold for ko/ru/ar detection to avoid misclassification (#658)
KorenKrita59ff67bacae2
diff --git a/openviking/session/memory_extractor.py b/openviking/session/memory_extractor.pyindex 46917f47d..dda6196bb 100644--- a/openviking/session/memory_extractor.py+++ b/openviking/session/memory_extractor.py@@ -161,7 +161,13 @@ class MemoryExtractor:if not user_text:return fallback- # Detect scripts that are largely language-unique first.+ # Detect scripts that are largely language-unique.+ # Require threshold to avoid misclassifying mixed-language texts+ # (e.g., Chinese with a single Cyrillic letter).+ total_chars = len(re.findall(r"\S", user_text))+ if total_chars == 0:+ return fallback+counts = {"ko": len(re.findall(r"[\uac00-\ud7af]", user_text)),"ru": len(re.findall(r"[\u0400-\u04ff]", user_text)),@@ -169,7 +175,8 @@ class MemoryExtractor:}detected, score = max(counts.items(), key=lambda item: item[1])- if score > 0:+ # Threshold: at least 2 chars AND at least 10% of non-whitespace chars+ if score >= 2 and score / total_chars >= 0.10:return detected# CJK disambiguation:diff --git a/tests/session/test_memory_extractor_language.py b/tests/session/test_memory_extractor_language.pyindex e9113d4b2..d1d980210 100644--- a/tests/session/test_memory_extractor_language.py+++ b/tests/session/test_memory_extractor_language.py@@ -86,3 +86,31 @@ def test_detect_output_language_japanese_with_more_han_than_kana():]language = MemoryExtractor._detect_output_language(messages, fallback_language="en")assert language == "ja"+++def test_detect_output_language_chinese_with_single_cyrillic():+ """Mixed Chinese with single Cyrillic char should be detected as Chinese, not Russian."""+ messages = [_msg("user", "\u8fd9\u662f\u4e2d\u6587 \u0414 \u518d\u7ee7\u7eed")]+ language = MemoryExtractor._detect_output_language(messages, fallback_language="en")+ assert language == "zh-CN"+++def test_detect_output_language_japanese_with_single_cyrillic():+ """Mixed Japanese with single Cyrillic char should be detected as Japanese, not Russian."""+ messages = [_msg("user", "\u3053\u308c\u306f\u65e5\u672c\u8a9e \u042f ")]+ language = MemoryExtractor._detect_output_language(messages, fallback_language="en")+ assert language == "ja"+++def test_detect_output_language_russian_with_threshold():+ """Russian text with sufficient Cyrillic chars should be detected as Russian."""+ messages = [_msg("user", "\u042d\u0442\u043e \u0440\u0443\u0441\u0441\u043a\u0438\u0439 \u0442\u0435\u043a\u0441\u0442")]+ language = MemoryExtractor._detect_output_language(messages, fallback_language="en")+ assert language == "ru"+++def test_detect_output_language_insufficient_cyrillic_fallback():+ """Text with only 1 Cyrillic char among Latin should fallback, not Russian."""+ messages = [_msg("user", "Hello \u0424 world")]+ language = MemoryExtractor._detect_output_language(messages, fallback_language="en")+ assert language == "en"