Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
Fix/language output (#2164)
fujiajie6663acc63c67004
diff --git a/openviking/session/memory/session_extract_context_provider.py b/openviking/session/memory/session_extract_context_provider.pyindex 731352580..67978cf87 100644--- a/openviking/session/memory/session_extract_context_provider.py+++ b/openviking/session/memory/session_extract_context_provider.py@@ -104,12 +104,16 @@ class SessionExtractContextProvider(ExtractContextProvider):from openviking.message.part import TextPartfrom openviking.session.memory.utils import resolve_output_language- text_parts = []+ user_text_parts = []+ all_text_parts = []for message in self.messages or []:for part in getattr(message, "parts", []):if isinstance(part, TextPart) and part.text:- text_parts.append(part.text)+ all_text_parts.append(part.text)+ if getattr(message, "role", "") == "user":+ user_text_parts.append(part.text)+ text_parts = user_text_parts or all_text_partsreturn resolve_output_language("\n".join(text_parts))def get_output_language(self) -> str:diff --git a/openviking/session/memory/utils/language.py b/openviking/session/memory/utils/language.pyindex bfcd3c158..2fa7d3f04 100644--- a/openviking/session/memory/utils/language.py+++ b/openviking/session/memory/utils/language.py@@ -4,7 +4,10 @@Language detection utilities."""+import locale+import osimport re+import timefrom typing import Callablefrom openviking_cli.utils import get_logger@@ -12,39 +15,264 @@ from openviking_cli.utils.config import get_openviking_configlogger = get_logger(__name__)+_SCRIPT_MIN_CHARS = 2+_SCRIPT_MIN_RATIO = 0.20+_JAPANESE_KANA_MIN_CHARS = 3+_STRONG_DOMINANT_MIN_CHARS = 10+_STRONG_DOMINANT_RATIO = 0.95+_PRIMARY_LANGUAGES = {"zh-CN", "en"}++_LATIN_STOPWORDS = {+ "en": set(+ "a an and are as be document for from in is of on please project that the this to user with".split()+ ),+ "it": set(+ "che con da del della di documento e il la le non per preferenze progetto questo questa un una utente".split()+ ),+ "fr": set(+ "avec ce cette de des document du et la le les pour préférences projet que un une utilisateur".split()+ ),+ "es": set(+ "con de del documento el esta este la las los para preferencias proyecto que un una usuario y".split()+ ),+ "de": set(+ "benutzer das der die diese dieser dokument ein eine für ist mit nicht projekt und zu".split()+ ),+ "pt": set(+ "a as com da de do documento e este esta o os para preferências preferencias projeto que um uma usuário usuario".split()+ ),+}++_LATIN_ACCENT_BONUSES = {+ "it": r"[àèéìòù]",+ "fr": r"[àâæçéèêëîïôœùûüÿ]",+ "es": r"[áéíóúüñ¿¡]",+ "de": r"[äöüß]",+ "pt": r"[áâãàçéêíóôõú]",+}+_LATIN_HINT_LANGUAGES = {"it", "fr", "es", "de", "pt"}++_LOCALE_LANGUAGE_PREFIXES = dict(+ zh="zh-CN", ja="ja", ko="ko", ru="ru", ar="ar",+ it="it", fr="fr", es="es", de="de", pt="pt", en="en",+ chinese="zh-CN", japanese="ja", korean="ko", russian="ru", arabic="ar",+ italian="it", french="fr", spanish="es", german="de", portuguese="pt", english="en",+)++# Use Timezone as a weak fallback signal.+_TIMEZONE_LANGUAGE_GROUPS = {+ "zh-CN": (+ "asia/shanghai", "asia/chongqing", "asia/harbin", "asia/urumqi",+ "asia/hong_kong", "asia/macau", "asia/taipei", "prc", "roc", "hongkong",+ "china standard time", "taipei standard time",+ ),+ "ja": ("asia/tokyo", "japan", "tokyo standard time"),+ "ko": ("asia/seoul", "rok", "korea standard time"),+ "ru": (+ "europe/moscow", "europe/kaliningrad", "asia/yekaterinburg", "asia/vladivostok",+ "russian standard time",+ ),+ "ar": (+ "asia/riyadh", "asia/dubai", "asia/qatar", "asia/kuwait",+ "asia/baghdad", "africa/cairo", "africa/algiers", "africa/tunis",+ "arab standard time", "arabian standard time", "egypt standard time",+ ),+ "it": ("europe/rome",),+ "fr": ("europe/paris",),+ "es": ("europe/madrid",),+ "de": ("europe/berlin",),+ "pt": ("europe/lisbon", "america/sao_paulo"),+ "en": (+ "america/new_york", "america/chicago", "america/denver", "america/los_angeles",+ "america/phoenix", "america/anchorage", "pacific/honolulu", "us/eastern",+ "us/central", "us/mountain", "us/pacific", "europe/london", "europe/dublin",+ "gb", "gb-eire", "america/toronto", "america/vancouver", "canada/eastern",+ "canada/pacific", "australia/sydney", "australia/melbourne",+ "australia/brisbane", "australia/perth", "pacific/auckland", "nz",+ "eastern standard time", "pacific standard time", "gmt standard time",+ ),+}++_TIMEZONE_LANGUAGE_HINTS = {+ timezone_name: language+ for language, timezone_names in _TIMEZONE_LANGUAGE_GROUPS.items()+ for timezone_name in timezone_names+}+++def _passes_threshold(count: int, total: int) -> bool:+ return count >= _SCRIPT_MIN_CHARS and total > 0 and count / total >= _SCRIPT_MIN_RATIO+++def _language_allowed_by_fallback(language: str, fallback_language: str) -> bool:+ return language in _PRIMARY_LANGUAGES or language == fallback_language+++def _is_strong_dominant(count: int, total: int) -> bool:+ return count >= _STRONG_DOMINANT_MIN_CHARS and total > 0 and count / total >= _STRONG_DOMINANT_RATIO+++def _language_from_locale_value(value: str) -> str:+ if not value:+ return ""+ normalized = value.split(":", 1)[0].split(".", 1)[0].split("@", 1)[0]+ normalized = normalized.strip().lower().replace("-", "_")+ if not normalized or normalized in {"c", "posix"}:+ return ""+ prefix = normalized.split("_", 1)[0].split(" ", 1)[0]+ return _LOCALE_LANGUAGE_PREFIXES.get(prefix, "")+++def _language_from_timezone_value(value: str) -> str:+ if not value:+ return ""+ normalized = value.strip().lower().lstrip(":")+ if not normalized or normalized == "local":+ return ""+ return _TIMEZONE_LANGUAGE_HINTS.get(normalized, "")+++def _language_from_local_timezone() -> str:+ try:+ path_parts = os.path.realpath("/etc/localtime").split(os.sep)+ for marker in ("zoneinfo", "zoneinfo.default"):+ if marker in path_parts:+ timezone_name = "/".join(path_parts[path_parts.index(marker) + 1 :])+ if language := _language_from_timezone_value(timezone_name):+ return language+ except Exception:+ pass++ for timezone_name in time.tzname:+ if language := _language_from_timezone_value(timezone_name or ""):+ return language+ return ""+++def _resolve_system_fallback_language(default_language: str = "en") -> str:+ """Resolve a weak fallback hint from system locale/timezone.++ The result is only used when text detection cannot identify a language.+ Explicit content and output_language_override still take precedence.+ """+ default = (default_language or "en").strip() or "en"+ english_locale_hint = ""++ for env_name in ("LC_ALL", "LC_MESSAGES", "LANGUAGE", "LANG"):+ language = _language_from_locale_value(os.environ.get(env_name, ""))+ if language and language != "en":+ return language+ if language == "en":+ english_locale_hint = language++ try:+ language = _language_from_locale_value(locale.getlocale()[0] or "")+ if language and language != "en":+ return language+ if language == "en":+ english_locale_hint = language+ except Exception:+ pass++ # Honor explicit TZ when set, mainly for Unix-like systems and CI/container environments.+ language = _language_from_timezone_value(os.environ.get("TZ", ""))+ if language:+ return language++ # Fall back to local timezone names; on Windows this may expose Standard Time names via time.tzname.+ language = _language_from_local_timezone()+ if language:+ return language++ if english_locale_hint:+ return english_locale_hint+ return default+++def _detect_latin_language(text: str, fallback_language: str) -> str:+ """Best-effort detector for common Latin-script languages.++ This intentionally stays conservative: if the signal is weak or tied, it+ uses English instead of guessing a non-English Latin language.+ """+ words = re.findall(r"[a-z\u00c0-\u024f]+", text.lower())+ if len(words) < 3:+ return "en"++ stopword_scores = {+ lang: sum(1 for word in words if word in stopwords)+ for lang, stopwords in _LATIN_STOPWORDS.items()+ }+ scores = dict(stopword_scores)++ lowered = text.lower()+ accent_hits = {}+ for lang, pattern in _LATIN_ACCENT_BONUSES.items():+ accent_hits[lang] = len(re.findall(pattern, lowered))+ scores[lang] += accent_hits[lang]++ language, score = max(scores.items(), key=lambda item: item[1])+ second_score = max((value for key, value in scores.items() if key != language), default=0)+ if language == "en" and score >= 2 and score > second_score:+ return "en"+ if language in _LATIN_HINT_LANGUAGES and len(words) >= 6:+ strong_hint = accent_hits.get(language, 0) > 0 or stopword_scores[language] >= 4+ strong_latin = len(words) >= 20 and score >= 6 and score >= scores.get("en", 0) + 4+ if (+ strong_hint+ and score >= 3+ and score >= scores.get("en", 0) + 2+ and (_language_allowed_by_fallback(language, fallback_language) or strong_latin)+ ):+ return language+ return "en"+def _detect_language_from_text(user_text: str, fallback_language: str) -> str:"""Internal shared helper to detect dominant language from text."""fallback = (fallback_language or "en").strip() or "en"- # return "zh-CN"-if not user_text:return fallback- # Detect scripts that are largely language-unique first.counts = {+ "zh-CN": len(re.findall(r"[\u4e00-\u9fff]", user_text)),+ "ja_kana": len(re.findall(r"[\u3040-\u30ff\u31f0-\u31ff\uff66-\uff9f]", user_text)),"ko": len(re.findall(r"[\uac00-\ud7af]", user_text)),"ru": len(re.findall(r"[\u0400-\u04ff]", user_text)),"ar": len(re.findall(r"[\u0600-\u06ff]", user_text)),+ "latin": len(re.findall(r"[A-Za-z\u00c0-\u024f]", user_text)),}+ signal_total = sum(counts.values())+ if signal_total == 0:+ return fallback- detected, score = max(counts.items(), key=lambda item: item[1])- if score > 0:- return detected-- # CJK disambiguation:- # - Japanese often includes Han characters too, so Han-count alone can- # misclassify Japanese as Chinese.- # - If any Kana is present, prioritize Japanese.- kana_count = len(re.findall(r"[\u3040-\u30ff\u31f0-\u31ff\uff66-\uff9f]", user_text))- han_count = len(re.findall(r"[\u4e00-\u9fff]", user_text))-- if kana_count > 0:+ japanese_total = counts["zh-CN"] + counts["ja_kana"]+ strong_japanese = (+ counts["ja_kana"] >= _STRONG_DOMINANT_MIN_CHARS+ and japanese_total / signal_total >= _STRONG_DOMINANT_RATIO+ and counts["ja_kana"] / japanese_total >= 0.30+ )+ if counts["ja_kana"] >= _JAPANESE_KANA_MIN_CHARS and (+ _language_allowed_by_fallback("ja", fallback) or strong_japanese+ ):return "ja"- if han_count > 0:- return "zh-CN"+ non_latin_candidates = {+ "zh-CN": counts["zh-CN"],+ "ko": counts["ko"],+ "ru": counts["ru"],+ "ar": counts["ar"],+ }+ language, score = max(non_latin_candidates.items(), key=lambda item: item[1])+ if _passes_threshold(score, signal_total) and (+ _language_allowed_by_fallback(language, fallback)+ or _is_strong_dominant(score, signal_total)+ ):+ return language++ if counts["latin"] > 0:+ return _detect_latin_language(user_text, fallback)return fallback@@ -65,9 +293,8 @@ def resolve_with_override(config, detect: Callable[[], str]) -> str:def resolve_output_language(text: str, config=None) -> str:"""Resolve output language from text, honoring config override before detection."""- result = resolve_with_override(config, lambda: _detect_language_from_text(text, "en"))- # result = '中文(必须遵守)'- return result+ fallback = _resolve_system_fallback_language("en")+ return resolve_with_override(config, lambda: _detect_language_from_text(text, fallback))def resolve_output_language_from_conversation(conversation: str, config=None) -> str:@@ -76,7 +303,10 @@ def resolve_output_language_from_conversation(conversation: str, config=None) ->When no override is set, uses `detect_language_from_conversation` whichscopes detection to user-role content only."""- return resolve_with_override(config, lambda: detect_language_from_conversation(conversation))+ fallback = _resolve_system_fallback_language("en")+ return resolve_with_override(+ config, lambda: detect_language_from_conversation(conversation, fallback)+ )def detect_language_from_conversation(conversation: str, fallback_language: str = "en") -> str:@@ -87,16 +317,24 @@ def detect_language_from_conversation(conversation: str, fallback_language: str"""fallback = (fallback_language or "en").strip() or "en"- # Try to extract user messages from conversation string- # Look for patterns like "[user]: ..." or "User: ..."+ # Try to extract user messages from conversation string.+ # Supports "[user]: ...", "User: ...", and indexed headers like+ # "[0][user][alice]: ...".user_lines = []for line in conversation.split("\n"):- line_lower = line.strip().lower()+ stripped = line.strip()+ line_lower = stripped.lower()if line_lower.startswith("[user]:") or line_lower.startswith("user:"):- # Extract content after the role marker- content = line.split(":", 1)[1].strip() if ":" in line else line.strip()+ content = stripped.split(":", 1)[1].strip() if ":" in stripped else strippedif content:user_lines.append(content)+ continue+ if ":" in stripped:+ header, content = stripped.split(":", 1)+ if re.search(r"\[\s*user\s*\]", header.lower()):+ content = content.strip()+ if content:+ user_lines.append(content)user_text = "\n".join(user_lines)diff --git a/openviking/session/memory_extractor.py b/openviking/session/memory_extractor.pyindex b09f1502e..179eba0e6 100644--- a/openviking/session/memory_extractor.py+++ b/openviking/session/memory_extractor.py@@ -153,6 +153,7 @@ class MemoryExtractor:text does not bias the target output language for stored memories."""fallback = (fallback_language or "en").strip() or "en"+ from openviking.session.memory.utils.language import _detect_language_from_textuser_text = "\n".join(str(getattr(m, "content", "") or "")@@ -163,37 +164,7 @@ class MemoryExtractor:if not user_text:return fallback- # Detect scripts that are largely language-unique.- # Require threshold to avoid misclassifying mixed-language texts- # (e.g., Chinese with a single Cyrillic letter).- total_chars = len(re.findall(r"\S", user_text))- if total_chars == 0:- return fallback-- counts = {- "ko": len(re.findall(r"[\uac00-\ud7af]", user_text)),- "ru": len(re.findall(r"[\u0400-\u04ff]", user_text)),- "ar": len(re.findall(r"[\u0600-\u06ff]", user_text)),- }-- detected, score = max(counts.items(), key=lambda item: item[1])- # Threshold: at least 2 chars AND at least 10% of non-whitespace chars- if score >= 2 and score / total_chars >= 0.10:- return detected-- # CJK disambiguation:- # - Japanese often includes Han characters too, so Han-count alone can- # misclassify Japanese as Chinese.- # - If any Kana is present, prioritize Japanese.- kana_count = len(re.findall(r"[\u3040-\u30ff\u31f0-\u31ff\uff66-\uff9f]", user_text))- han_count = len(re.findall(r"[\u4e00-\u9fff]", user_text))-- if kana_count > 0:- return "ja"- if han_count > 0:- return "zh-CN"-- return fallback+ return _detect_language_from_text(user_text, fallback)def _format_message_with_parts(self, msg) -> str:"""格式化单条消息,包含文本和工具调用"""@@ -278,12 +249,18 @@ class MemoryExtractor:logger.warning("No formatted messages, returning empty list")return []- from openviking.session.memory.utils.language import resolve_with_override+ from openviking.session.memory.utils.language import (+ _resolve_system_fallback_language,+ resolve_with_override,+ )config = get_openviking_config()output_language = resolve_with_override(config,- lambda: self._detect_output_language(messages, fallback_language="en"),+ lambda: self._detect_output_language(+ messages,+ fallback_language=_resolve_system_fallback_language("en"),+ ),)history_summary = str(context.get("summary") or "")diff --git a/tests/session/memory/test_memory_react_system_prompt.py b/tests/session/memory/test_memory_react_system_prompt.pyindex 6009706a7..be2af4bae 100644--- a/tests/session/memory/test_memory_react_system_prompt.py+++ b/tests/session/memory/test_memory_react_system_prompt.py@@ -122,3 +122,21 @@ class TestSkillToolCallExposure:provider = SessionExtractContextProvider(messages=messages)assert provider._detect_language() == "en"++ def test_detect_language_prefers_user_text_over_assistant_text(self):+ messages = [+ Message(+ id="m1",+ role="user",+ parts=[TextPart("请把记忆保持为中文,继续优化。")],+ ),+ Message(+ id="m2",+ role="assistant",+ parts=[TextPart("한국어 응답이 섞였습니다")],+ ),+ ]++ provider = SessionExtractContextProvider(messages=messages)++ assert provider._detect_language() == "zh-CN"diff --git a/tests/session/test_memory_extractor_language.py b/tests/session/test_memory_extractor_language.pyindex 46bb00478..a290ff23e 100644--- a/tests/session/test_memory_extractor_language.py+++ b/tests/session/test_memory_extractor_language.py@@ -65,7 +65,7 @@ def test_detect_output_language_japanese_kana_and_kanji():"\u4eca\u65e5\u306f\u65b0\u6a5f\u80fd\u306e\u8a2d\u8a08\u3092\u9032\u3081\u307e\u3059",)]- language = MemoryExtractor._detect_output_language(messages, fallback_language="en")+ language = MemoryExtractor._detect_output_language(messages, fallback_language="ja")assert language == "ja"@@ -73,7 +73,7 @@ def test_detect_output_language_chinese_han_only():messages = [_msg("user", "\u4eca\u5929\u7ee7\u7eed\u4f18\u5316\u8bb0\u5fc6\u62bd\u53d6\u6a21\u5757")]- language = MemoryExtractor._detect_output_language(messages, fallback_language="en")+ language = MemoryExtractor._detect_output_language(messages, fallback_language="ja")assert language == "zh-CN"@@ -84,21 +84,21 @@ def test_detect_output_language_japanese_with_more_han_than_kana():"\u65b0\u6a5f\u80fd\u8a2d\u8a08\u306e\u65b9\u91dd\u3092\u78ba\u8a8d\u3057\u307e\u3059",)]- language = MemoryExtractor._detect_output_language(messages, fallback_language="en")+ language = MemoryExtractor._detect_output_language(messages, fallback_language="ja")assert language == "ja"def test_detect_output_language_chinese_with_single_cyrillic():"""Mixed Chinese with single Cyrillic char should be detected as Chinese, not Russian."""messages = [_msg("user", "\u8fd9\u662f\u4e2d\u6587 \u0414 \u518d\u7ee7\u7eed")]- language = MemoryExtractor._detect_output_language(messages, fallback_language="en")+ language = MemoryExtractor._detect_output_language(messages, fallback_language="ru")assert language == "zh-CN"def test_detect_output_language_japanese_with_single_cyrillic():"""Mixed Japanese with single Cyrillic char should be detected as Japanese, not Russian."""messages = [_msg("user", "\u3053\u308c\u306f\u65e5\u672c\u8a9e \u042f ")]- language = MemoryExtractor._detect_output_language(messages, fallback_language="en")+ language = MemoryExtractor._detect_output_language(messages, fallback_language="ja")assert language == "ja"@@ -110,7 +110,7 @@ def test_detect_output_language_russian_with_threshold():"\u042d\u0442\u043e \u0440\u0443\u0441\u0441\u043a\u0438\u0439 \u0442\u0435\u043a\u0441\u0442",)]- language = MemoryExtractor._detect_output_language(messages, fallback_language="en")+ language = MemoryExtractor._detect_output_language(messages, fallback_language="ru")assert language == "ru"@@ -119,3 +119,14 @@ def test_detect_output_language_insufficient_cyrillic_fallback():messages = [_msg("user", "Hello \u0424 world")]language = MemoryExtractor._detect_output_language(messages, fallback_language="en")assert language == "en"+++def test_detect_output_language_italian():+ messages = [+ _msg(+ "user",+ "Questo documento descrive le preferenze dell utente e il progetto da completare.",+ )+ ]+ language = MemoryExtractor._detect_output_language(messages, fallback_language="it")+ assert language == "it"diff --git a/tests/storage/test_semantic_processor_language.py b/tests/storage/test_semantic_processor_language.pyindex dc441d472..fd3b00f91 100644--- a/tests/storage/test_semantic_processor_language.py+++ b/tests/storage/test_semantic_processor_language.py@@ -2,6 +2,7 @@# SPDX-License-Identifier: Apache-2.0import asyncio+import osimport reimport tempfilefrom pathlib import Path@@ -12,6 +13,9 @@ import pytestfrom openviking.prompts import render_promptfrom openviking.session.memory.utils.language import (_detect_language_from_text,+ _language_from_locale_value,+ _language_from_timezone_value,+ _resolve_system_fallback_language,resolve_output_language,resolve_output_language_from_conversation,)@@ -32,22 +36,65 @@ class TestLanguageDetection:def test_detect_language_japanese(self):text = "これは日本語のドキュメントです"- language = _detect_language_from_text(text, fallback_language="en")+ language = _detect_language_from_text(text, fallback_language="ja")+ assert language == "ja"++ def test_detect_language_kanji_heavy_japanese(self):+ text = "明日は会議です"+ language = _detect_language_from_text(text, fallback_language="ja")+ assert language == "ja"++ def test_japanese_text_uses_system_fallback_when_system_is_not_japanese(self):+ text = "明日は会議です"+ language = _detect_language_from_text(text, fallback_language="zh-CN")+ assert language == "zh-CN"++ def test_strong_japanese_text_can_override_system_fallback(self):+ text = (+ "今日は新しい機能の設計を進めます。明日の会議で方針を確認します。"+ "そのあとで実装とテストをまとめます。"+ )+ language = _detect_language_from_text(text, fallback_language="zh-CN")assert language == "ja"+ def test_japanese_title_does_not_override_chinese_fallback(self):+ text = "请记住我最近在读《ノルウェイの森》,后面继续用中文讨论这个内容"+ language = _detect_language_from_text(text, fallback_language="zh-CN")+ assert language == "zh-CN"++ def test_single_kana_does_not_override_chinese(self):+ text = "这是中文の测试"+ language = _detect_language_from_text(text, fallback_language="en")+ assert language == "zh-CN"+def test_detect_language_korean(self):text = "이것은 한국어 문서입니다"- language = _detect_language_from_text(text, fallback_language="en")+ language = _detect_language_from_text(text, fallback_language="ko")+ assert language == "ko"++ def test_strong_korean_text_can_override_system_fallback(self):+ text = "이것은 한국어로 작성된 긴 문서입니다 사용자의 선호와 프로젝트 내용을 기록합니다"+ language = _detect_language_from_text(text, fallback_language="zh-CN")assert language == "ko"def test_detect_language_russian(self):text = "Это русский документ"- language = _detect_language_from_text(text, fallback_language="en")+ language = _detect_language_from_text(text, fallback_language="ru")+ assert language == "ru"++ def test_strong_russian_text_can_override_system_fallback(self):+ text = "Это русский документ для проверки памяти пользователя и настроек проекта"+ language = _detect_language_from_text(text, fallback_language="zh-CN")assert language == "ru"def test_detect_language_arabic(self):text = "هذا مستند باللغة العربية"- language = _detect_language_from_text(text, fallback_language="en")+ language = _detect_language_from_text(text, fallback_language="ar")+ assert language == "ar"++ def test_strong_arabic_text_can_override_system_fallback(self):+ text = "هذا مستند عربي طويل لتسجيل تفضيلات المستخدم ومعلومات المشروع"+ language = _detect_language_from_text(text, fallback_language="zh-CN")assert language == "ar"def test_detect_language_empty_text(self):@@ -60,6 +107,51 @@ class TestLanguageDetection:language = _detect_language_from_text(text, fallback_language="en")assert language == "zh-CN"+ def test_detect_language_chinese_with_single_korean_char(self):+ text = "这是中文需求,继续优化记忆。한"+ language = _detect_language_from_text(text, fallback_language="en")+ assert language == "zh-CN"++ def test_detect_language_chinese_with_single_cyrillic_char(self):+ text = "这是中文需求,继续优化记忆。Д"+ language = _detect_language_from_text(text, fallback_language="en")+ assert language == "zh-CN"++ def test_detect_language_english_with_single_korean_char(self):+ text = "Please optimize memory extraction 한"+ language = _detect_language_from_text(text, fallback_language="en")+ assert language == "en"++ def test_detect_language_italian(self):+ text = (+ "Questo documento descrive le preferenze dell utente "+ "e il progetto da completare."+ )+ language = _detect_language_from_text(text, fallback_language="it")+ assert language == "it"++ def test_strong_italian_text_can_override_system_fallback(self):+ text = (+ "Questo documento descrive le preferenze dell utente e il progetto da completare. "+ "Il contenuto include le decisioni, le attività, la priorità e una nota finale."+ )+ language = _detect_language_from_text(text, fallback_language="zh-CN")+ assert language == "it"++ @pytest.mark.parametrize(+ "text,expected",+ [+ ("project document user data model profile", "en"),+ ("Ce document décrit les préférences de l utilisateur et le projet à terminer.", "fr"),+ ("Este documento describe las preferencias del usuario y el proyecto para completar.", "es"),+ ("Dieses Dokument beschreibt die Präferenzen der Benutzer und das Projekt.", "de"),+ ("Este documento descreve as preferências do usuário e o projeto para completar.", "pt"),+ ],+ )+ def test_detect_latin_language_conservatively(self, text, expected):+ language = _detect_language_from_text(text, fallback_language=expected)+ assert language == expected+class TestLanguageFlow:"""语言检测 + 模板渲染流程测试。"""@@ -77,7 +169,7 @@ class TestLanguageFlow:)def test_language_detection_to_template_flow(self, lang, content, file_name):"""语言检测 -> output_language 注入模板 -> prompt 包含语言指令"""- detected_lang = _detect_language_from_text(content, fallback_language="en")+ detected_lang = _detect_language_from_text(content, fallback_language=lang)assert detected_lang == lang, f"Expected {lang}, got {detected_lang}"prompt = render_prompt(@@ -100,7 +192,7 @@ class TestOverviewGenerationFlow:)def test_overview_generation_language_flow(self, lang, file_summaries):"""目录摘要 -> 语言检测 -> overview 模板"""- detected_lang = _detect_language_from_text(file_summaries, fallback_language="en")+ detected_lang = _detect_language_from_text(file_summaries, fallback_language=lang)assert detected_lang == langprompt = render_prompt(@@ -185,6 +277,15 @@ def _verify_content_language(text: str, expected_lang: str) -> bool:class TestGenerateTextSummaryOutputLanguage:"""端到端测试:验证 _generate_text_summary 生成的内容语言是否符合预期。"""+ _LANGUAGE_LOCALE = {+ "zh-CN": "zh_CN.UTF-8",+ "en": "en_US.UTF-8",+ "ja": "ja_JP.UTF-8",+ "ko": "ko_KR.UTF-8",+ "ru": "ru_RU.UTF-8",+ "ar": "ar_SA.UTF-8",+ }+@pytest.fixturedef temp_multilang_files(self):"""创建包含多种语言内容的临时测试文件。"""@@ -256,31 +357,33 @@ class TestGenerateTextSummaryOutputLanguage:mock_viking_fs = self._create_mock_viking_fs(content)mock_config = self._create_mock_config(mock_vlm)- with patch(+ with patch.dict(+ os.environ,+ {"LC_ALL": self._LANGUAGE_LOCALE[expected_lang]},+ ), patch("openviking.storage.queuefs.semantic_processor.get_viking_fs",return_value=mock_viking_fs,+ ), patch(+ "openviking.storage.queuefs.semantic_processor.get_openviking_config",+ return_value=mock_config,):- with patch(- "openviking.storage.queuefs.semantic_processor.get_openviking_config",- return_value=mock_config,- ):- processor = SemanticProcessor()- processor._current_ctx = MagicMock()-- result = await processor._generate_text_summary(- file_path=temp_multilang_files[file_key],- file_name=file_name,- llm_sem=asyncio.Semaphore(1),- )-- prompt_sent = mock_vlm.prompts_received[0]- assert f"Output Language: {expected_lang}" in prompt_sent, (- f"{file_name}: Prompt missing Output Language: {expected_lang}"- )-- assert _verify_content_language(result["summary"], expected_lang), (- f"{file_name}: Content language mismatch. Expected {expected_lang}, got: {result['summary']}"- )+ processor = SemanticProcessor()+ processor._current_ctx = MagicMock()++ result = await processor._generate_text_summary(+ file_path=temp_multilang_files[file_key],+ file_name=file_name,+ llm_sem=asyncio.Semaphore(1),+ )++ prompt_sent = mock_vlm.prompts_received[0]+ assert f"Output Language: {expected_lang}" in prompt_sent, (+ f"{file_name}: Prompt missing Output Language: {expected_lang}"+ )++ assert _verify_content_language(result["summary"], expected_lang), (+ f"{file_name}: Content language mismatch. Expected {expected_lang}, got: {result['summary']}"+ )@pytest.mark.asyncio@pytest.mark.parametrize(@@ -298,29 +401,31 @@ class TestGenerateTextSummaryOutputLanguage:mock_viking_fs = self._create_mock_viking_fs(content)mock_config = self._create_mock_config(mock_vlm)- with patch(+ with patch.dict(+ os.environ,+ {"LC_ALL": self._LANGUAGE_LOCALE[expected_lang]},+ ), patch("openviking.storage.queuefs.semantic_processor.get_viking_fs",return_value=mock_viking_fs,+ ), patch(+ "openviking.storage.queuefs.semantic_processor.get_openviking_config",+ return_value=mock_config,):- with patch(- "openviking.storage.queuefs.semantic_processor.get_openviking_config",- return_value=mock_config,- ):- processor = SemanticProcessor()- processor._current_ctx = MagicMock()+ processor = SemanticProcessor()+ processor._current_ctx = MagicMock()- result = await processor._generate_text_summary(- file_path=f"/tmp/{file_name}",- file_name=file_name,- llm_sem=asyncio.Semaphore(1),- )+ result = await processor._generate_text_summary(+ file_path=f"/tmp/{file_name}",+ file_name=file_name,+ llm_sem=asyncio.Semaphore(1),+ )- prompt_sent = mock_vlm.prompts_received[0]- assert f"Output Language: {expected_lang}" in prompt_sent+ prompt_sent = mock_vlm.prompts_received[0]+ assert f"Output Language: {expected_lang}" in prompt_sent- assert _verify_content_language(result["summary"], expected_lang), (- f"{file_name}: Content language mismatch. Expected {expected_lang}, got: {result['summary']}"- )+ assert _verify_content_language(result["summary"], expected_lang), (+ f"{file_name}: Content language mismatch. Expected {expected_lang}, got: {result['summary']}"+ )class TestOutputLanguageOverride:@@ -334,10 +439,11 @@ class TestOutputLanguageOverride:def test_override_unset_detects_from_content(self):config = self._make_config(override="")- result = resolve_output_language("これは日本語のテキストです", config=config)+ with patch.dict(os.environ, {"LC_ALL": "ja_JP.UTF-8"}):+ result = resolve_output_language("これは日本語のテキストです", config=config)assert result == "ja"- def test_override_unset_uses_fallback_for_latin_text(self):+ def test_override_unset_uses_english_for_latin_text(self):config = self._make_config(override="", fallback="en")result = resolve_output_language("Plain English text with no special scripts", config=config@@ -356,9 +462,114 @@ class TestOutputLanguageOverride:def test_override_whitespace_treated_as_unset(self):config = self._make_config(override=" ")- result = resolve_output_language("これは日本語のテキストです", config=config)+ with patch.dict(os.environ, {"LC_ALL": "ja_JP.UTF-8"}):+ result = resolve_output_language("これは日本語のテキストです", config=config)+ assert result == "ja"++ def test_locale_hint_used_when_content_has_no_language_signal(self):+ config = self._make_config(override="")+ with patch.dict(os.environ, {"LC_ALL": "zh_CN.UTF-8"}, clear=True):+ result = resolve_output_language("12345 ---", config=config)+ assert result == "zh-CN"++ @pytest.mark.parametrize(+ "locale_value,expected",+ [+ ("Chinese_China.936", "zh-CN"),+ ("Chinese (Simplified)_China.936", "zh-CN"),+ ("English_United States.1252", "en"),+ ],+ )+ def test_windows_locale_hint_values(self, locale_value, expected):+ assert _language_from_locale_value(locale_value) == expected++ def test_timezone_hint_used_when_locale_hint_absent(self):+ config = self._make_config(override="")+ with patch.dict(os.environ, {"TZ": "Asia/Tokyo"}, clear=True):+ result = resolve_output_language("12345 ---", config=config)assert result == "ja"+ @pytest.mark.parametrize(+ "timezone_value,expected",+ [+ ("China Standard Time", "zh-CN"),+ ("Tokyo Standard Time", "ja"),+ ("Eastern Standard Time", "en"),+ ],+ )+ def test_windows_timezone_hint_values(self, timezone_value, expected):+ assert _language_from_timezone_value(timezone_value) == expected++ def test_timezone_hint_overrides_english_locale_for_weak_fallback(self):+ with patch.dict(+ os.environ,+ {"LC_ALL": "en_US.UTF-8", "TZ": "Asia/Shanghai"},+ clear=True,+ ):+ assert _resolve_system_fallback_language("en") == "zh-CN"++ def test_non_english_locale_hint_wins_over_timezone(self):+ with patch.dict(+ os.environ,+ {"LC_ALL": "ja_JP.UTF-8", "TZ": "Asia/Shanghai"},+ clear=True,+ ):+ assert _resolve_system_fallback_language("en") == "ja"++ def test_local_timezone_hint_used_when_tz_env_absent(self):+ with patch.dict(os.environ, {}, clear=True), patch(+ "openviking.session.memory.utils.language.locale.getlocale",+ return_value=("C", "UTF-8"),+ ), patch(+ "openviking.session.memory.utils.language.os.path.realpath",+ return_value="/usr/share/zoneinfo.default/Asia/Shanghai",+ ):+ assert _resolve_system_fallback_language("en") == "zh-CN"++ def test_english_timezone_hint_used_when_locale_hint_absent(self):+ config = self._make_config(override="")+ with patch.dict(os.environ, {"TZ": "America/New_York"}, clear=True):+ result = resolve_output_language("12345 ---", config=config)+ assert result == "en"++ def test_arabic_timezone_hint_used_when_locale_hint_absent(self):+ config = self._make_config(override="")+ with patch.dict(os.environ, {"TZ": "Asia/Riyadh"}, clear=True):+ result = resolve_output_language("12345 ---", config=config)+ assert result == "ar"++ def test_content_language_wins_over_locale_hint(self):+ config = self._make_config(override="")+ with patch.dict(os.environ, {"LC_ALL": "zh_CN.UTF-8"}, clear=True):+ result = resolve_output_language(+ "This is an English document for testing language detection",+ config=config,+ )+ assert result == "en"++ def test_english_content_wins_over_chinese_timezone_hint(self):+ config = self._make_config(override="")+ with patch.dict(+ os.environ,+ {"LC_ALL": "en_US.UTF-8", "TZ": "Asia/Shanghai"},+ clear=True,+ ):+ result = resolve_output_language(+ "This is an English document for testing language detection",+ config=config,+ )+ assert result == "en"++ def test_short_latin_content_wins_over_chinese_timezone_hint(self):+ config = self._make_config(override="")+ with patch.dict(+ os.environ,+ {"LC_ALL": "en_US.UTF-8", "TZ": "Asia/Shanghai"},+ clear=True,+ ):+ result = resolve_output_language("Use Vim", config=config)+ assert result == "en"+def test_conversation_override_set_bypasses_detection(self):config = self._make_config(override="en")conversation = "[user]: これは日本語のメッセージです\n[assistant]: reply"@@ -368,5 +579,12 @@ class TestOutputLanguageOverride:def test_conversation_override_unset_detects_from_user_content(self):config = self._make_config(override="")conversation = "[user]: これは日本語のメッセージです\n[assistant]: reply"- result = resolve_output_language_from_conversation(conversation, config=config)+ with patch.dict(os.environ, {"LC_ALL": "ja_JP.UTF-8"}):+ result = resolve_output_language_from_conversation(conversation, config=config)assert result == "ja"++ def test_indexed_conversation_detects_user_content(self):+ config = self._make_config(override="")+ conversation = "[0][user][alice]: 请使用中文\n[1][assistant][bot]: 한국어 응답"+ result = resolve_output_language_from_conversation(conversation, config=config)+ assert result == "zh-CN"