Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

Fix/language output (#2164)

fujiajie6663acc63c67004
diff --git a/openviking/session/memory/session_extract_context_provider.py b/openviking/session/memory/session_extract_context_provider.py
index 731352580..67978cf87 100644
--- a/openviking/session/memory/session_extract_context_provider.py
+++ b/openviking/session/memory/session_extract_context_provider.py
@@ -104,12 +104,16 @@ class SessionExtractContextProvider(ExtractContextProvider):
from openviking.message.part import TextPart
from openviking.session.memory.utils import resolve_output_language
- text_parts = []
+ user_text_parts = []
+ all_text_parts = []
for message in self.messages or []:
for part in getattr(message, "parts", []):
if isinstance(part, TextPart) and part.text:
- text_parts.append(part.text)
+ all_text_parts.append(part.text)
+ if getattr(message, "role", "") == "user":
+ user_text_parts.append(part.text)
+ text_parts = user_text_parts or all_text_parts
return resolve_output_language("\n".join(text_parts))
def get_output_language(self) -> str:
diff --git a/openviking/session/memory/utils/language.py b/openviking/session/memory/utils/language.py
index bfcd3c158..2fa7d3f04 100644
--- a/openviking/session/memory/utils/language.py
+++ b/openviking/session/memory/utils/language.py
@@ -4,7 +4,10 @@
Language detection utilities.
"""
+import locale
+import os
import re
+import time
from typing import Callable
from openviking_cli.utils import get_logger
@@ -12,39 +15,264 @@ from openviking_cli.utils.config import get_openviking_config
logger = get_logger(__name__)
+_SCRIPT_MIN_CHARS = 2
+_SCRIPT_MIN_RATIO = 0.20
+_JAPANESE_KANA_MIN_CHARS = 3
+_STRONG_DOMINANT_MIN_CHARS = 10
+_STRONG_DOMINANT_RATIO = 0.95
+_PRIMARY_LANGUAGES = {"zh-CN", "en"}
+
+_LATIN_STOPWORDS = {
+ "en": set(
+ "a an and are as be document for from in is of on please project that the this to user with".split()
+ ),
+ "it": set(
+ "che con da del della di documento e il la le non per preferenze progetto questo questa un una utente".split()
+ ),
+ "fr": set(
+ "avec ce cette de des document du et la le les pour préférences projet que un une utilisateur".split()
+ ),
+ "es": set(
+ "con de del documento el esta este la las los para preferencias proyecto que un una usuario y".split()
+ ),
+ "de": set(
+ "benutzer das der die diese dieser dokument ein eine für ist mit nicht projekt und zu".split()
+ ),
+ "pt": set(
+ "a as com da de do documento e este esta o os para preferências preferencias projeto que um uma usuário usuario".split()
+ ),
+}
+
+_LATIN_ACCENT_BONUSES = {
+ "it": r"[àèéìòù]",
+ "fr": r"[àâæçéèêëîïôœùûüÿ]",
+ "es": r"[áéíóúüñ¿¡]",
+ "de": r"[äöüß]",
+ "pt": r"[áâãàçéêíóôõú]",
+}
+_LATIN_HINT_LANGUAGES = {"it", "fr", "es", "de", "pt"}
+
+_LOCALE_LANGUAGE_PREFIXES = dict(
+ zh="zh-CN", ja="ja", ko="ko", ru="ru", ar="ar",
+ it="it", fr="fr", es="es", de="de", pt="pt", en="en",
+ chinese="zh-CN", japanese="ja", korean="ko", russian="ru", arabic="ar",
+ italian="it", french="fr", spanish="es", german="de", portuguese="pt", english="en",
+)
+
+# Use Timezone as a weak fallback signal.
+_TIMEZONE_LANGUAGE_GROUPS = {
+ "zh-CN": (
+ "asia/shanghai", "asia/chongqing", "asia/harbin", "asia/urumqi",
+ "asia/hong_kong", "asia/macau", "asia/taipei", "prc", "roc", "hongkong",
+ "china standard time", "taipei standard time",
+ ),
+ "ja": ("asia/tokyo", "japan", "tokyo standard time"),
+ "ko": ("asia/seoul", "rok", "korea standard time"),
+ "ru": (
+ "europe/moscow", "europe/kaliningrad", "asia/yekaterinburg", "asia/vladivostok",
+ "russian standard time",
+ ),
+ "ar": (
+ "asia/riyadh", "asia/dubai", "asia/qatar", "asia/kuwait",
+ "asia/baghdad", "africa/cairo", "africa/algiers", "africa/tunis",
+ "arab standard time", "arabian standard time", "egypt standard time",
+ ),
+ "it": ("europe/rome",),
+ "fr": ("europe/paris",),
+ "es": ("europe/madrid",),
+ "de": ("europe/berlin",),
+ "pt": ("europe/lisbon", "america/sao_paulo"),
+ "en": (
+ "america/new_york", "america/chicago", "america/denver", "america/los_angeles",
+ "america/phoenix", "america/anchorage", "pacific/honolulu", "us/eastern",
+ "us/central", "us/mountain", "us/pacific", "europe/london", "europe/dublin",
+ "gb", "gb-eire", "america/toronto", "america/vancouver", "canada/eastern",
+ "canada/pacific", "australia/sydney", "australia/melbourne",
+ "australia/brisbane", "australia/perth", "pacific/auckland", "nz",
+ "eastern standard time", "pacific standard time", "gmt standard time",
+ ),
+}
+
+_TIMEZONE_LANGUAGE_HINTS = {
+ timezone_name: language
+ for language, timezone_names in _TIMEZONE_LANGUAGE_GROUPS.items()
+ for timezone_name in timezone_names
+}
+
+
+def _passes_threshold(count: int, total: int) -> bool:
+ return count >= _SCRIPT_MIN_CHARS and total > 0 and count / total >= _SCRIPT_MIN_RATIO
+
+
+def _language_allowed_by_fallback(language: str, fallback_language: str) -> bool:
+ return language in _PRIMARY_LANGUAGES or language == fallback_language
+
+
+def _is_strong_dominant(count: int, total: int) -> bool:
+ return count >= _STRONG_DOMINANT_MIN_CHARS and total > 0 and count / total >= _STRONG_DOMINANT_RATIO
+
+
+def _language_from_locale_value(value: str) -> str:
+ if not value:
+ return ""
+ normalized = value.split(":", 1)[0].split(".", 1)[0].split("@", 1)[0]
+ normalized = normalized.strip().lower().replace("-", "_")
+ if not normalized or normalized in {"c", "posix"}:
+ return ""
+ prefix = normalized.split("_", 1)[0].split(" ", 1)[0]
+ return _LOCALE_LANGUAGE_PREFIXES.get(prefix, "")
+
+
+def _language_from_timezone_value(value: str) -> str:
+ if not value:
+ return ""
+ normalized = value.strip().lower().lstrip(":")
+ if not normalized or normalized == "local":
+ return ""
+ return _TIMEZONE_LANGUAGE_HINTS.get(normalized, "")
+
+
+def _language_from_local_timezone() -> str:
+ try:
+ path_parts = os.path.realpath("/etc/localtime").split(os.sep)
+ for marker in ("zoneinfo", "zoneinfo.default"):
+ if marker in path_parts:
+ timezone_name = "/".join(path_parts[path_parts.index(marker) + 1 :])
+ if language := _language_from_timezone_value(timezone_name):
+ return language
+ except Exception:
+ pass
+
+ for timezone_name in time.tzname:
+ if language := _language_from_timezone_value(timezone_name or ""):
+ return language
+ return ""
+
+
+def _resolve_system_fallback_language(default_language: str = "en") -> str:
+ """Resolve a weak fallback hint from system locale/timezone.
+
+ The result is only used when text detection cannot identify a language.
+ Explicit content and output_language_override still take precedence.
+ """
+ default = (default_language or "en").strip() or "en"
+ english_locale_hint = ""
+
+ for env_name in ("LC_ALL", "LC_MESSAGES", "LANGUAGE", "LANG"):
+ language = _language_from_locale_value(os.environ.get(env_name, ""))
+ if language and language != "en":
+ return language
+ if language == "en":
+ english_locale_hint = language
+
+ try:
+ language = _language_from_locale_value(locale.getlocale()[0] or "")
+ if language and language != "en":
+ return language
+ if language == "en":
+ english_locale_hint = language
+ except Exception:
+ pass
+
+ # Honor explicit TZ when set, mainly for Unix-like systems and CI/container environments.
+ language = _language_from_timezone_value(os.environ.get("TZ", ""))
+ if language:
+ return language
+
+ # Fall back to local timezone names; on Windows this may expose Standard Time names via time.tzname.
+ language = _language_from_local_timezone()
+ if language:
+ return language
+
+ if english_locale_hint:
+ return english_locale_hint
+ return default
+
+
+def _detect_latin_language(text: str, fallback_language: str) -> str:
+ """Best-effort detector for common Latin-script languages.
+
+ This intentionally stays conservative: if the signal is weak or tied, it
+ uses English instead of guessing a non-English Latin language.
+ """
+ words = re.findall(r"[a-z\u00c0-\u024f]+", text.lower())
+ if len(words) < 3:
+ return "en"
+
+ stopword_scores = {
+ lang: sum(1 for word in words if word in stopwords)
+ for lang, stopwords in _LATIN_STOPWORDS.items()
+ }
+ scores = dict(stopword_scores)
+
+ lowered = text.lower()
+ accent_hits = {}
+ for lang, pattern in _LATIN_ACCENT_BONUSES.items():
+ accent_hits[lang] = len(re.findall(pattern, lowered))
+ scores[lang] += accent_hits[lang]
+
+ language, score = max(scores.items(), key=lambda item: item[1])
+ second_score = max((value for key, value in scores.items() if key != language), default=0)
+ if language == "en" and score >= 2 and score > second_score:
+ return "en"
+ if language in _LATIN_HINT_LANGUAGES and len(words) >= 6:
+ strong_hint = accent_hits.get(language, 0) > 0 or stopword_scores[language] >= 4
+ strong_latin = len(words) >= 20 and score >= 6 and score >= scores.get("en", 0) + 4
+ if (
+ strong_hint
+ and score >= 3
+ and score >= scores.get("en", 0) + 2
+ and (_language_allowed_by_fallback(language, fallback_language) or strong_latin)
+ ):
+ return language
+ return "en"
+
def _detect_language_from_text(user_text: str, fallback_language: str) -> str:
"""Internal shared helper to detect dominant language from text."""
fallback = (fallback_language or "en").strip() or "en"
- # return "zh-CN"
-
if not user_text:
return fallback
- # Detect scripts that are largely language-unique first.
counts = {
+ "zh-CN": len(re.findall(r"[\u4e00-\u9fff]", user_text)),
+ "ja_kana": len(re.findall(r"[\u3040-\u30ff\u31f0-\u31ff\uff66-\uff9f]", user_text)),
"ko": len(re.findall(r"[\uac00-\ud7af]", user_text)),
"ru": len(re.findall(r"[\u0400-\u04ff]", user_text)),
"ar": len(re.findall(r"[\u0600-\u06ff]", user_text)),
+ "latin": len(re.findall(r"[A-Za-z\u00c0-\u024f]", user_text)),
}
+ signal_total = sum(counts.values())
+ if signal_total == 0:
+ return fallback
- detected, score = max(counts.items(), key=lambda item: item[1])
- if score > 0:
- return detected
-
- # CJK disambiguation:
- # - Japanese often includes Han characters too, so Han-count alone can
- # misclassify Japanese as Chinese.
- # - If any Kana is present, prioritize Japanese.
- kana_count = len(re.findall(r"[\u3040-\u30ff\u31f0-\u31ff\uff66-\uff9f]", user_text))
- han_count = len(re.findall(r"[\u4e00-\u9fff]", user_text))
-
- if kana_count > 0:
+ japanese_total = counts["zh-CN"] + counts["ja_kana"]
+ strong_japanese = (
+ counts["ja_kana"] >= _STRONG_DOMINANT_MIN_CHARS
+ and japanese_total / signal_total >= _STRONG_DOMINANT_RATIO
+ and counts["ja_kana"] / japanese_total >= 0.30
+ )
+ if counts["ja_kana"] >= _JAPANESE_KANA_MIN_CHARS and (
+ _language_allowed_by_fallback("ja", fallback) or strong_japanese
+ ):
return "ja"
- if han_count > 0:
- return "zh-CN"
+ non_latin_candidates = {
+ "zh-CN": counts["zh-CN"],
+ "ko": counts["ko"],
+ "ru": counts["ru"],
+ "ar": counts["ar"],
+ }
+ language, score = max(non_latin_candidates.items(), key=lambda item: item[1])
+ if _passes_threshold(score, signal_total) and (
+ _language_allowed_by_fallback(language, fallback)
+ or _is_strong_dominant(score, signal_total)
+ ):
+ return language
+
+ if counts["latin"] > 0:
+ return _detect_latin_language(user_text, fallback)
return fallback
@@ -65,9 +293,8 @@ def resolve_with_override(config, detect: Callable[[], str]) -> str:
def resolve_output_language(text: str, config=None) -> str:
"""Resolve output language from text, honoring config override before detection."""
- result = resolve_with_override(config, lambda: _detect_language_from_text(text, "en"))
- # result = '中文(必须遵守)'
- return result
+ fallback = _resolve_system_fallback_language("en")
+ return resolve_with_override(config, lambda: _detect_language_from_text(text, fallback))
def resolve_output_language_from_conversation(conversation: str, config=None) -> str:
@@ -76,7 +303,10 @@ def resolve_output_language_from_conversation(conversation: str, config=None) ->
When no override is set, uses `detect_language_from_conversation` which
scopes detection to user-role content only.
"""
- return resolve_with_override(config, lambda: detect_language_from_conversation(conversation))
+ fallback = _resolve_system_fallback_language("en")
+ return resolve_with_override(
+ config, lambda: detect_language_from_conversation(conversation, fallback)
+ )
def detect_language_from_conversation(conversation: str, fallback_language: str = "en") -> str:
@@ -87,16 +317,24 @@ def detect_language_from_conversation(conversation: str, fallback_language: str
"""
fallback = (fallback_language or "en").strip() or "en"
- # Try to extract user messages from conversation string
- # Look for patterns like "[user]: ..." or "User: ..."
+ # Try to extract user messages from conversation string.
+ # Supports "[user]: ...", "User: ...", and indexed headers like
+ # "[0][user][alice]: ...".
user_lines = []
for line in conversation.split("\n"):
- line_lower = line.strip().lower()
+ stripped = line.strip()
+ line_lower = stripped.lower()
if line_lower.startswith("[user]:") or line_lower.startswith("user:"):
- # Extract content after the role marker
- content = line.split(":", 1)[1].strip() if ":" in line else line.strip()
+ content = stripped.split(":", 1)[1].strip() if ":" in stripped else stripped
if content:
user_lines.append(content)
+ continue
+ if ":" in stripped:
+ header, content = stripped.split(":", 1)
+ if re.search(r"\[\s*user\s*\]", header.lower()):
+ content = content.strip()
+ if content:
+ user_lines.append(content)
user_text = "\n".join(user_lines)
diff --git a/openviking/session/memory_extractor.py b/openviking/session/memory_extractor.py
index b09f1502e..179eba0e6 100644
--- a/openviking/session/memory_extractor.py
+++ b/openviking/session/memory_extractor.py
@@ -153,6 +153,7 @@ class MemoryExtractor:
text does not bias the target output language for stored memories.
"""
fallback = (fallback_language or "en").strip() or "en"
+ from openviking.session.memory.utils.language import _detect_language_from_text
user_text = "\n".join(
str(getattr(m, "content", "") or "")
@@ -163,37 +164,7 @@ class MemoryExtractor:
if not user_text:
return fallback
- # Detect scripts that are largely language-unique.
- # Require threshold to avoid misclassifying mixed-language texts
- # (e.g., Chinese with a single Cyrillic letter).
- total_chars = len(re.findall(r"\S", user_text))
- if total_chars == 0:
- return fallback
-
- counts = {
- "ko": len(re.findall(r"[\uac00-\ud7af]", user_text)),
- "ru": len(re.findall(r"[\u0400-\u04ff]", user_text)),
- "ar": len(re.findall(r"[\u0600-\u06ff]", user_text)),
- }
-
- detected, score = max(counts.items(), key=lambda item: item[1])
- # Threshold: at least 2 chars AND at least 10% of non-whitespace chars
- if score >= 2 and score / total_chars >= 0.10:
- return detected
-
- # CJK disambiguation:
- # - Japanese often includes Han characters too, so Han-count alone can
- # misclassify Japanese as Chinese.
- # - If any Kana is present, prioritize Japanese.
- kana_count = len(re.findall(r"[\u3040-\u30ff\u31f0-\u31ff\uff66-\uff9f]", user_text))
- han_count = len(re.findall(r"[\u4e00-\u9fff]", user_text))
-
- if kana_count > 0:
- return "ja"
- if han_count > 0:
- return "zh-CN"
-
- return fallback
+ return _detect_language_from_text(user_text, fallback)
def _format_message_with_parts(self, msg) -> str:
"""格式化单条消息,包含文本和工具调用"""
@@ -278,12 +249,18 @@ class MemoryExtractor:
logger.warning("No formatted messages, returning empty list")
return []
- from openviking.session.memory.utils.language import resolve_with_override
+ from openviking.session.memory.utils.language import (
+ _resolve_system_fallback_language,
+ resolve_with_override,
+ )
config = get_openviking_config()
output_language = resolve_with_override(
config,
- lambda: self._detect_output_language(messages, fallback_language="en"),
+ lambda: self._detect_output_language(
+ messages,
+ fallback_language=_resolve_system_fallback_language("en"),
+ ),
)
history_summary = str(context.get("summary") or "")
diff --git a/tests/session/memory/test_memory_react_system_prompt.py b/tests/session/memory/test_memory_react_system_prompt.py
index 6009706a7..be2af4bae 100644
--- a/tests/session/memory/test_memory_react_system_prompt.py
+++ b/tests/session/memory/test_memory_react_system_prompt.py
@@ -122,3 +122,21 @@ class TestSkillToolCallExposure:
provider = SessionExtractContextProvider(messages=messages)
assert provider._detect_language() == "en"
+
+ def test_detect_language_prefers_user_text_over_assistant_text(self):
+ messages = [
+ Message(
+ id="m1",
+ role="user",
+ parts=[TextPart("请把记忆保持为中文,继续优化。")],
+ ),
+ Message(
+ id="m2",
+ role="assistant",
+ parts=[TextPart("한국어 응답이 섞였습니다")],
+ ),
+ ]
+
+ provider = SessionExtractContextProvider(messages=messages)
+
+ assert provider._detect_language() == "zh-CN"
diff --git a/tests/session/test_memory_extractor_language.py b/tests/session/test_memory_extractor_language.py
index 46bb00478..a290ff23e 100644
--- a/tests/session/test_memory_extractor_language.py
+++ b/tests/session/test_memory_extractor_language.py
@@ -65,7 +65,7 @@ def test_detect_output_language_japanese_kana_and_kanji():
"\u4eca\u65e5\u306f\u65b0\u6a5f\u80fd\u306e\u8a2d\u8a08\u3092\u9032\u3081\u307e\u3059",
)
]
- language = MemoryExtractor._detect_output_language(messages, fallback_language="en")
+ language = MemoryExtractor._detect_output_language(messages, fallback_language="ja")
assert language == "ja"
@@ -73,7 +73,7 @@ def test_detect_output_language_chinese_han_only():
messages = [
_msg("user", "\u4eca\u5929\u7ee7\u7eed\u4f18\u5316\u8bb0\u5fc6\u62bd\u53d6\u6a21\u5757")
]
- language = MemoryExtractor._detect_output_language(messages, fallback_language="en")
+ language = MemoryExtractor._detect_output_language(messages, fallback_language="ja")
assert language == "zh-CN"
@@ -84,21 +84,21 @@ def test_detect_output_language_japanese_with_more_han_than_kana():
"\u65b0\u6a5f\u80fd\u8a2d\u8a08\u306e\u65b9\u91dd\u3092\u78ba\u8a8d\u3057\u307e\u3059",
)
]
- language = MemoryExtractor._detect_output_language(messages, fallback_language="en")
+ language = MemoryExtractor._detect_output_language(messages, fallback_language="ja")
assert language == "ja"
def test_detect_output_language_chinese_with_single_cyrillic():
"""Mixed Chinese with single Cyrillic char should be detected as Chinese, not Russian."""
messages = [_msg("user", "\u8fd9\u662f\u4e2d\u6587 \u0414 \u518d\u7ee7\u7eed")]
- language = MemoryExtractor._detect_output_language(messages, fallback_language="en")
+ language = MemoryExtractor._detect_output_language(messages, fallback_language="ru")
assert language == "zh-CN"
def test_detect_output_language_japanese_with_single_cyrillic():
"""Mixed Japanese with single Cyrillic char should be detected as Japanese, not Russian."""
messages = [_msg("user", "\u3053\u308c\u306f\u65e5\u672c\u8a9e \u042f ")]
- language = MemoryExtractor._detect_output_language(messages, fallback_language="en")
+ language = MemoryExtractor._detect_output_language(messages, fallback_language="ja")
assert language == "ja"
@@ -110,7 +110,7 @@ def test_detect_output_language_russian_with_threshold():
"\u042d\u0442\u043e \u0440\u0443\u0441\u0441\u043a\u0438\u0439 \u0442\u0435\u043a\u0441\u0442",
)
]
- language = MemoryExtractor._detect_output_language(messages, fallback_language="en")
+ language = MemoryExtractor._detect_output_language(messages, fallback_language="ru")
assert language == "ru"
@@ -119,3 +119,14 @@ def test_detect_output_language_insufficient_cyrillic_fallback():
messages = [_msg("user", "Hello \u0424 world")]
language = MemoryExtractor._detect_output_language(messages, fallback_language="en")
assert language == "en"
+
+
+def test_detect_output_language_italian():
+ messages = [
+ _msg(
+ "user",
+ "Questo documento descrive le preferenze dell utente e il progetto da completare.",
+ )
+ ]
+ language = MemoryExtractor._detect_output_language(messages, fallback_language="it")
+ assert language == "it"
diff --git a/tests/storage/test_semantic_processor_language.py b/tests/storage/test_semantic_processor_language.py
index dc441d472..fd3b00f91 100644
--- a/tests/storage/test_semantic_processor_language.py
+++ b/tests/storage/test_semantic_processor_language.py
@@ -2,6 +2,7 @@
# SPDX-License-Identifier: Apache-2.0
import asyncio
+import os
import re
import tempfile
from pathlib import Path
@@ -12,6 +13,9 @@ import pytest
from openviking.prompts import render_prompt
from openviking.session.memory.utils.language import (
_detect_language_from_text,
+ _language_from_locale_value,
+ _language_from_timezone_value,
+ _resolve_system_fallback_language,
resolve_output_language,
resolve_output_language_from_conversation,
)
@@ -32,22 +36,65 @@ class TestLanguageDetection:
def test_detect_language_japanese(self):
text = "これは日本語のドキュメントです"
- language = _detect_language_from_text(text, fallback_language="en")
+ language = _detect_language_from_text(text, fallback_language="ja")
+ assert language == "ja"
+
+ def test_detect_language_kanji_heavy_japanese(self):
+ text = "明日は会議です"
+ language = _detect_language_from_text(text, fallback_language="ja")
+ assert language == "ja"
+
+ def test_japanese_text_uses_system_fallback_when_system_is_not_japanese(self):
+ text = "明日は会議です"
+ language = _detect_language_from_text(text, fallback_language="zh-CN")
+ assert language == "zh-CN"
+
+ def test_strong_japanese_text_can_override_system_fallback(self):
+ text = (
+ "今日は新しい機能の設計を進めます。明日の会議で方針を確認します。"
+ "そのあとで実装とテストをまとめます。"
+ )
+ language = _detect_language_from_text(text, fallback_language="zh-CN")
assert language == "ja"
+ def test_japanese_title_does_not_override_chinese_fallback(self):
+ text = "请记住我最近在读《ノルウェイの森》,后面继续用中文讨论这个内容"
+ language = _detect_language_from_text(text, fallback_language="zh-CN")
+ assert language == "zh-CN"
+
+ def test_single_kana_does_not_override_chinese(self):
+ text = "这是中文の测试"
+ language = _detect_language_from_text(text, fallback_language="en")
+ assert language == "zh-CN"
+
def test_detect_language_korean(self):
text = "이것은 한국어 문서입니다"
- language = _detect_language_from_text(text, fallback_language="en")
+ language = _detect_language_from_text(text, fallback_language="ko")
+ assert language == "ko"
+
+ def test_strong_korean_text_can_override_system_fallback(self):
+ text = "이것은 한국어로 작성된 긴 문서입니다 사용자의 선호와 프로젝트 내용을 기록합니다"
+ language = _detect_language_from_text(text, fallback_language="zh-CN")
assert language == "ko"
def test_detect_language_russian(self):
text = "Это русский документ"
- language = _detect_language_from_text(text, fallback_language="en")
+ language = _detect_language_from_text(text, fallback_language="ru")
+ assert language == "ru"
+
+ def test_strong_russian_text_can_override_system_fallback(self):
+ text = "Это русский документ для проверки памяти пользователя и настроек проекта"
+ language = _detect_language_from_text(text, fallback_language="zh-CN")
assert language == "ru"
def test_detect_language_arabic(self):
text = "هذا مستند باللغة العربية"
- language = _detect_language_from_text(text, fallback_language="en")
+ language = _detect_language_from_text(text, fallback_language="ar")
+ assert language == "ar"
+
+ def test_strong_arabic_text_can_override_system_fallback(self):
+ text = "هذا مستند عربي طويل لتسجيل تفضيلات المستخدم ومعلومات المشروع"
+ language = _detect_language_from_text(text, fallback_language="zh-CN")
assert language == "ar"
def test_detect_language_empty_text(self):
@@ -60,6 +107,51 @@ class TestLanguageDetection:
language = _detect_language_from_text(text, fallback_language="en")
assert language == "zh-CN"
+ def test_detect_language_chinese_with_single_korean_char(self):
+ text = "这是中文需求,继续优化记忆。한"
+ language = _detect_language_from_text(text, fallback_language="en")
+ assert language == "zh-CN"
+
+ def test_detect_language_chinese_with_single_cyrillic_char(self):
+ text = "这是中文需求,继续优化记忆。Д"
+ language = _detect_language_from_text(text, fallback_language="en")
+ assert language == "zh-CN"
+
+ def test_detect_language_english_with_single_korean_char(self):
+ text = "Please optimize memory extraction 한"
+ language = _detect_language_from_text(text, fallback_language="en")
+ assert language == "en"
+
+ def test_detect_language_italian(self):
+ text = (
+ "Questo documento descrive le preferenze dell utente "
+ "e il progetto da completare."
+ )
+ language = _detect_language_from_text(text, fallback_language="it")
+ assert language == "it"
+
+ def test_strong_italian_text_can_override_system_fallback(self):
+ text = (
+ "Questo documento descrive le preferenze dell utente e il progetto da completare. "
+ "Il contenuto include le decisioni, le attività, la priorità e una nota finale."
+ )
+ language = _detect_language_from_text(text, fallback_language="zh-CN")
+ assert language == "it"
+
+ @pytest.mark.parametrize(
+ "text,expected",
+ [
+ ("project document user data model profile", "en"),
+ ("Ce document décrit les préférences de l utilisateur et le projet à terminer.", "fr"),
+ ("Este documento describe las preferencias del usuario y el proyecto para completar.", "es"),
+ ("Dieses Dokument beschreibt die Präferenzen der Benutzer und das Projekt.", "de"),
+ ("Este documento descreve as preferências do usuário e o projeto para completar.", "pt"),
+ ],
+ )
+ def test_detect_latin_language_conservatively(self, text, expected):
+ language = _detect_language_from_text(text, fallback_language=expected)
+ assert language == expected
+
class TestLanguageFlow:
"""语言检测 + 模板渲染流程测试。"""
@@ -77,7 +169,7 @@ class TestLanguageFlow:
)
def test_language_detection_to_template_flow(self, lang, content, file_name):
"""语言检测 -> output_language 注入模板 -> prompt 包含语言指令"""
- detected_lang = _detect_language_from_text(content, fallback_language="en")
+ detected_lang = _detect_language_from_text(content, fallback_language=lang)
assert detected_lang == lang, f"Expected {lang}, got {detected_lang}"
prompt = render_prompt(
@@ -100,7 +192,7 @@ class TestOverviewGenerationFlow:
)
def test_overview_generation_language_flow(self, lang, file_summaries):
"""目录摘要 -> 语言检测 -> overview 模板"""
- detected_lang = _detect_language_from_text(file_summaries, fallback_language="en")
+ detected_lang = _detect_language_from_text(file_summaries, fallback_language=lang)
assert detected_lang == lang
prompt = render_prompt(
@@ -185,6 +277,15 @@ def _verify_content_language(text: str, expected_lang: str) -> bool:
class TestGenerateTextSummaryOutputLanguage:
"""端到端测试:验证 _generate_text_summary 生成的内容语言是否符合预期。"""
+ _LANGUAGE_LOCALE = {
+ "zh-CN": "zh_CN.UTF-8",
+ "en": "en_US.UTF-8",
+ "ja": "ja_JP.UTF-8",
+ "ko": "ko_KR.UTF-8",
+ "ru": "ru_RU.UTF-8",
+ "ar": "ar_SA.UTF-8",
+ }
+
@pytest.fixture
def temp_multilang_files(self):
"""创建包含多种语言内容的临时测试文件。"""
@@ -256,31 +357,33 @@ class TestGenerateTextSummaryOutputLanguage:
mock_viking_fs = self._create_mock_viking_fs(content)
mock_config = self._create_mock_config(mock_vlm)
- with patch(
+ with patch.dict(
+ os.environ,
+ {"LC_ALL": self._LANGUAGE_LOCALE[expected_lang]},
+ ), patch(
"openviking.storage.queuefs.semantic_processor.get_viking_fs",
return_value=mock_viking_fs,
+ ), patch(
+ "openviking.storage.queuefs.semantic_processor.get_openviking_config",
+ return_value=mock_config,
):
- with patch(
- "openviking.storage.queuefs.semantic_processor.get_openviking_config",
- return_value=mock_config,
- ):
- processor = SemanticProcessor()
- processor._current_ctx = MagicMock()
-
- result = await processor._generate_text_summary(
- file_path=temp_multilang_files[file_key],
- file_name=file_name,
- llm_sem=asyncio.Semaphore(1),
- )
-
- prompt_sent = mock_vlm.prompts_received[0]
- assert f"Output Language: {expected_lang}" in prompt_sent, (
- f"{file_name}: Prompt missing Output Language: {expected_lang}"
- )
-
- assert _verify_content_language(result["summary"], expected_lang), (
- f"{file_name}: Content language mismatch. Expected {expected_lang}, got: {result['summary']}"
- )
+ processor = SemanticProcessor()
+ processor._current_ctx = MagicMock()
+
+ result = await processor._generate_text_summary(
+ file_path=temp_multilang_files[file_key],
+ file_name=file_name,
+ llm_sem=asyncio.Semaphore(1),
+ )
+
+ prompt_sent = mock_vlm.prompts_received[0]
+ assert f"Output Language: {expected_lang}" in prompt_sent, (
+ f"{file_name}: Prompt missing Output Language: {expected_lang}"
+ )
+
+ assert _verify_content_language(result["summary"], expected_lang), (
+ f"{file_name}: Content language mismatch. Expected {expected_lang}, got: {result['summary']}"
+ )
@pytest.mark.asyncio
@pytest.mark.parametrize(
@@ -298,29 +401,31 @@ class TestGenerateTextSummaryOutputLanguage:
mock_viking_fs = self._create_mock_viking_fs(content)
mock_config = self._create_mock_config(mock_vlm)
- with patch(
+ with patch.dict(
+ os.environ,
+ {"LC_ALL": self._LANGUAGE_LOCALE[expected_lang]},
+ ), patch(
"openviking.storage.queuefs.semantic_processor.get_viking_fs",
return_value=mock_viking_fs,
+ ), patch(
+ "openviking.storage.queuefs.semantic_processor.get_openviking_config",
+ return_value=mock_config,
):
- with patch(
- "openviking.storage.queuefs.semantic_processor.get_openviking_config",
- return_value=mock_config,
- ):
- processor = SemanticProcessor()
- processor._current_ctx = MagicMock()
+ processor = SemanticProcessor()
+ processor._current_ctx = MagicMock()
- result = await processor._generate_text_summary(
- file_path=f"/tmp/{file_name}",
- file_name=file_name,
- llm_sem=asyncio.Semaphore(1),
- )
+ result = await processor._generate_text_summary(
+ file_path=f"/tmp/{file_name}",
+ file_name=file_name,
+ llm_sem=asyncio.Semaphore(1),
+ )
- prompt_sent = mock_vlm.prompts_received[0]
- assert f"Output Language: {expected_lang}" in prompt_sent
+ prompt_sent = mock_vlm.prompts_received[0]
+ assert f"Output Language: {expected_lang}" in prompt_sent
- assert _verify_content_language(result["summary"], expected_lang), (
- f"{file_name}: Content language mismatch. Expected {expected_lang}, got: {result['summary']}"
- )
+ assert _verify_content_language(result["summary"], expected_lang), (
+ f"{file_name}: Content language mismatch. Expected {expected_lang}, got: {result['summary']}"
+ )
class TestOutputLanguageOverride:
@@ -334,10 +439,11 @@ class TestOutputLanguageOverride:
def test_override_unset_detects_from_content(self):
config = self._make_config(override="")
- result = resolve_output_language("これは日本語のテキストです", config=config)
+ with patch.dict(os.environ, {"LC_ALL": "ja_JP.UTF-8"}):
+ result = resolve_output_language("これは日本語のテキストです", config=config)
assert result == "ja"
- def test_override_unset_uses_fallback_for_latin_text(self):
+ def test_override_unset_uses_english_for_latin_text(self):
config = self._make_config(override="", fallback="en")
result = resolve_output_language(
"Plain English text with no special scripts", config=config
@@ -356,9 +462,114 @@ class TestOutputLanguageOverride:
def test_override_whitespace_treated_as_unset(self):
config = self._make_config(override=" ")
- result = resolve_output_language("これは日本語のテキストです", config=config)
+ with patch.dict(os.environ, {"LC_ALL": "ja_JP.UTF-8"}):
+ result = resolve_output_language("これは日本語のテキストです", config=config)
+ assert result == "ja"
+
+ def test_locale_hint_used_when_content_has_no_language_signal(self):
+ config = self._make_config(override="")
+ with patch.dict(os.environ, {"LC_ALL": "zh_CN.UTF-8"}, clear=True):
+ result = resolve_output_language("12345 ---", config=config)
+ assert result == "zh-CN"
+
+ @pytest.mark.parametrize(
+ "locale_value,expected",
+ [
+ ("Chinese_China.936", "zh-CN"),
+ ("Chinese (Simplified)_China.936", "zh-CN"),
+ ("English_United States.1252", "en"),
+ ],
+ )
+ def test_windows_locale_hint_values(self, locale_value, expected):
+ assert _language_from_locale_value(locale_value) == expected
+
+ def test_timezone_hint_used_when_locale_hint_absent(self):
+ config = self._make_config(override="")
+ with patch.dict(os.environ, {"TZ": "Asia/Tokyo"}, clear=True):
+ result = resolve_output_language("12345 ---", config=config)
assert result == "ja"
+ @pytest.mark.parametrize(
+ "timezone_value,expected",
+ [
+ ("China Standard Time", "zh-CN"),
+ ("Tokyo Standard Time", "ja"),
+ ("Eastern Standard Time", "en"),
+ ],
+ )
+ def test_windows_timezone_hint_values(self, timezone_value, expected):
+ assert _language_from_timezone_value(timezone_value) == expected
+
+ def test_timezone_hint_overrides_english_locale_for_weak_fallback(self):
+ with patch.dict(
+ os.environ,
+ {"LC_ALL": "en_US.UTF-8", "TZ": "Asia/Shanghai"},
+ clear=True,
+ ):
+ assert _resolve_system_fallback_language("en") == "zh-CN"
+
+ def test_non_english_locale_hint_wins_over_timezone(self):
+ with patch.dict(
+ os.environ,
+ {"LC_ALL": "ja_JP.UTF-8", "TZ": "Asia/Shanghai"},
+ clear=True,
+ ):
+ assert _resolve_system_fallback_language("en") == "ja"
+
+ def test_local_timezone_hint_used_when_tz_env_absent(self):
+ with patch.dict(os.environ, {}, clear=True), patch(
+ "openviking.session.memory.utils.language.locale.getlocale",
+ return_value=("C", "UTF-8"),
+ ), patch(
+ "openviking.session.memory.utils.language.os.path.realpath",
+ return_value="/usr/share/zoneinfo.default/Asia/Shanghai",
+ ):
+ assert _resolve_system_fallback_language("en") == "zh-CN"
+
+ def test_english_timezone_hint_used_when_locale_hint_absent(self):
+ config = self._make_config(override="")
+ with patch.dict(os.environ, {"TZ": "America/New_York"}, clear=True):
+ result = resolve_output_language("12345 ---", config=config)
+ assert result == "en"
+
+ def test_arabic_timezone_hint_used_when_locale_hint_absent(self):
+ config = self._make_config(override="")
+ with patch.dict(os.environ, {"TZ": "Asia/Riyadh"}, clear=True):
+ result = resolve_output_language("12345 ---", config=config)
+ assert result == "ar"
+
+ def test_content_language_wins_over_locale_hint(self):
+ config = self._make_config(override="")
+ with patch.dict(os.environ, {"LC_ALL": "zh_CN.UTF-8"}, clear=True):
+ result = resolve_output_language(
+ "This is an English document for testing language detection",
+ config=config,
+ )
+ assert result == "en"
+
+ def test_english_content_wins_over_chinese_timezone_hint(self):
+ config = self._make_config(override="")
+ with patch.dict(
+ os.environ,
+ {"LC_ALL": "en_US.UTF-8", "TZ": "Asia/Shanghai"},
+ clear=True,
+ ):
+ result = resolve_output_language(
+ "This is an English document for testing language detection",
+ config=config,
+ )
+ assert result == "en"
+
+ def test_short_latin_content_wins_over_chinese_timezone_hint(self):
+ config = self._make_config(override="")
+ with patch.dict(
+ os.environ,
+ {"LC_ALL": "en_US.UTF-8", "TZ": "Asia/Shanghai"},
+ clear=True,
+ ):
+ result = resolve_output_language("Use Vim", config=config)
+ assert result == "en"
+
def test_conversation_override_set_bypasses_detection(self):
config = self._make_config(override="en")
conversation = "[user]: これは日本語のメッセージです\n[assistant]: reply"
@@ -368,5 +579,12 @@ class TestOutputLanguageOverride:
def test_conversation_override_unset_detects_from_user_content(self):
config = self._make_config(override="")
conversation = "[user]: これは日本語のメッセージです\n[assistant]: reply"
- result = resolve_output_language_from_conversation(conversation, config=config)
+ with patch.dict(os.environ, {"LC_ALL": "ja_JP.UTF-8"}):
+ result = resolve_output_language_from_conversation(conversation, config=config)
assert result == "ja"
+
+ def test_indexed_conversation_detects_user_content(self):
+ config = self._make_config(override="")
+ conversation = "[0][user][alice]: 请使用中文\n[1][assistant][bot]: 한국어 응답"
+ result = resolve_output_language_from_conversation(conversation, config=config)
+ assert result == "zh-CN"