Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(language): add threshold for ko/ru/ar detection to avoid misclassification (#658)

KorenKrita59ff67bacae2
diff --git a/openviking/session/memory_extractor.py b/openviking/session/memory_extractor.py
index 46917f47d..dda6196bb 100644
--- a/openviking/session/memory_extractor.py
+++ b/openviking/session/memory_extractor.py
@@ -161,7 +161,13 @@ class MemoryExtractor:
if not user_text:
return fallback
- # Detect scripts that are largely language-unique first.
+ # Detect scripts that are largely language-unique.
+ # Require threshold to avoid misclassifying mixed-language texts
+ # (e.g., Chinese with a single Cyrillic letter).
+ total_chars = len(re.findall(r"\S", user_text))
+ if total_chars == 0:
+ return fallback
+
counts = {
"ko": len(re.findall(r"[\uac00-\ud7af]", user_text)),
"ru": len(re.findall(r"[\u0400-\u04ff]", user_text)),
@@ -169,7 +175,8 @@ class MemoryExtractor:
}
detected, score = max(counts.items(), key=lambda item: item[1])
- if score > 0:
+ # Threshold: at least 2 chars AND at least 10% of non-whitespace chars
+ if score >= 2 and score / total_chars >= 0.10:
return detected
# CJK disambiguation:
diff --git a/tests/session/test_memory_extractor_language.py b/tests/session/test_memory_extractor_language.py
index e9113d4b2..d1d980210 100644
--- a/tests/session/test_memory_extractor_language.py
+++ b/tests/session/test_memory_extractor_language.py
@@ -86,3 +86,31 @@ def test_detect_output_language_japanese_with_more_han_than_kana():
]
language = MemoryExtractor._detect_output_language(messages, fallback_language="en")
assert language == "ja"
+
+
+def test_detect_output_language_chinese_with_single_cyrillic():
+ """Mixed Chinese with single Cyrillic char should be detected as Chinese, not Russian."""
+ messages = [_msg("user", "\u8fd9\u662f\u4e2d\u6587 \u0414 \u518d\u7ee7\u7eed")]
+ language = MemoryExtractor._detect_output_language(messages, fallback_language="en")
+ assert language == "zh-CN"
+
+
+def test_detect_output_language_japanese_with_single_cyrillic():
+ """Mixed Japanese with single Cyrillic char should be detected as Japanese, not Russian."""
+ messages = [_msg("user", "\u3053\u308c\u306f\u65e5\u672c\u8a9e \u042f ")]
+ language = MemoryExtractor._detect_output_language(messages, fallback_language="en")
+ assert language == "ja"
+
+
+def test_detect_output_language_russian_with_threshold():
+ """Russian text with sufficient Cyrillic chars should be detected as Russian."""
+ messages = [_msg("user", "\u042d\u0442\u043e \u0440\u0443\u0441\u0441\u043a\u0438\u0439 \u0442\u0435\u043a\u0441\u0442")]
+ language = MemoryExtractor._detect_output_language(messages, fallback_language="en")
+ assert language == "ru"
+
+
+def test_detect_output_language_insufficient_cyrillic_fallback():
+ """Text with only 1 Cyrillic char among Latin should fallback, not Russian."""
+ messages = [_msg("user", "Hello \u0424 world")]
+ language = MemoryExtractor._detect_output_language(messages, fallback_language="en")
+ assert language == "en"