Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(semantic): add output_language_override to pin summary/overview language (#1607)

Brian Le7e347f2baaef
diff --git a/openviking/session/memory/session_extract_context_provider.py b/openviking/session/memory/session_extract_context_provider.py
index c7fb02e8f..4c6bef942 100644
--- a/openviking/session/memory/session_extract_context_provider.py
+++ b/openviking/session/memory/session_extract_context_provider.py
@@ -54,12 +54,10 @@ class SessionExtractContextProvider(ExtractContextProvider):
def _detect_language(self) -> str:
"""检测输出语言"""
- from openviking.session.memory.utils import detect_language_from_conversation
+ from openviking.session.memory.utils import resolve_output_language_from_conversation
conversation = self._assemble_conversation(self.messages)
- config = get_openviking_config()
- fallback_language = (config.language_fallback or "en").strip() or "en"
- return detect_language_from_conversation(conversation, fallback_language=fallback_language)
+ return resolve_output_language_from_conversation(conversation)
def instruction(self) -> str:
output_language = self._output_language
diff --git a/openviking/session/memory/utils/__init__.py b/openviking/session/memory/utils/__init__.py
index 91524efe9..4b6f47dcd 100644
--- a/openviking/session/memory/utils/__init__.py
+++ b/openviking/session/memory/utils/__init__.py
@@ -23,6 +23,9 @@ from openviking.session.memory.utils.json_parser import (
)
from openviking.session.memory.utils.language import (
detect_language_from_conversation,
+ resolve_output_language,
+ resolve_output_language_from_conversation,
+ resolve_with_override,
)
from openviking.session.memory.utils.messages import (
parse_memory_file_with_fields,
@@ -56,6 +59,9 @@ __all__ = [
"truncate_content",
# Language
"detect_language_from_conversation",
+ "resolve_output_language",
+ "resolve_output_language_from_conversation",
+ "resolve_with_override",
# Messages
"pretty_print_messages",
"parse_memory_file_with_fields",
diff --git a/openviking/session/memory/utils/language.py b/openviking/session/memory/utils/language.py
index e68cff788..a26154643 100644
--- a/openviking/session/memory/utils/language.py
+++ b/openviking/session/memory/utils/language.py
@@ -5,8 +5,10 @@ Language detection utilities.
"""
import re
+from typing import Callable
from openviking_cli.utils import get_logger
+from openviking_cli.utils.config import get_openviking_config
logger = get_logger(__name__)
@@ -15,7 +17,7 @@ def _detect_language_from_text(user_text: str, fallback_language: str) -> str:
"""Internal shared helper to detect dominant language from text."""
fallback = (fallback_language or "en").strip() or "en"
- #return "zh-CN"
+ # return "zh-CN"
if not user_text:
return fallback
@@ -46,6 +48,35 @@ def _detect_language_from_text(user_text: str, fallback_language: str) -> str:
return fallback
+def resolve_with_override(config, detect: Callable[[], str]) -> str:
+ """Return config override if set, else call `detect()`.
+
+ The callable returns the detected output language, letting callers choose
+ the detector (text vs conversation vs messages) without duplicating the
+ override resolution logic.
+ """
+ if config is None:
+ config = get_openviking_config()
+ override = (getattr(config, "output_language_override", None) or "").strip()
+ if override:
+ return override
+ return detect()
+
+
+def resolve_output_language(text: str, config=None) -> str:
+ """Resolve output language from text, honoring config override before detection."""
+ return resolve_with_override(config, lambda: _detect_language_from_text(text, "en"))
+
+
+def resolve_output_language_from_conversation(conversation: str, config=None) -> str:
+ """Resolve output language from a conversation, honoring config override.
+
+ When no override is set, uses `detect_language_from_conversation` which
+ scopes detection to user-role content only.
+ """
+ return resolve_with_override(config, lambda: detect_language_from_conversation(conversation))
+
+
def detect_language_from_conversation(conversation: str, fallback_language: str = "en") -> str:
"""Detect dominant language from user messages in conversation.
diff --git a/openviking/session/memory_extractor.py b/openviking/session/memory_extractor.py
index 1ea96434a..de765097e 100644
--- a/openviking/session/memory_extractor.py
+++ b/openviking/session/memory_extractor.py
@@ -278,10 +278,12 @@ class MemoryExtractor:
logger.warning("No formatted messages, returning empty list")
return []
+ from openviking.session.memory.utils.language import resolve_with_override
+
config = get_openviking_config()
- fallback_language = (config.language_fallback or "en").strip() or "en"
- output_language = self._detect_output_language(
- messages, fallback_language=fallback_language
+ output_language = resolve_with_override(
+ config,
+ lambda fb: self._detect_output_language(messages, fallback_language=fb),
)
history_summary = str(context.get("summary") or "")
diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py
index 3f4217a3c..32984fe1e 100644
--- a/openviking/storage/queuefs/semantic_processor.py
+++ b/openviking/storage/queuefs/semantic_processor.py
@@ -872,10 +872,9 @@ class SemanticProcessor(DequeueHandlerBase):
logger.warning("VLM not available, using empty summary")
return {"name": file_name, "summary": ""}
- from openviking.session.memory.utils.language import _detect_language_from_text
+ from openviking.session.memory.utils.language import resolve_output_language
- fallback_language = (get_openviking_config().language_fallback or "en").strip() or "en"
- output_language = _detect_language_from_text(content, fallback_language)
+ output_language = resolve_output_language(content)
# Detect file type and select appropriate prompt
file_type = self._detect_file_type(file_name)
@@ -1079,9 +1078,7 @@ class SemanticProcessor(DequeueHandlerBase):
logger.warning("VLM not available, using default overview")
return f"# {dir_uri.split('/')[-1]}\n\n[Directory overview is not ready]"
- from openviking.session.memory.utils.language import _detect_language_from_text
-
- fallback_language = (config.language_fallback or "en").strip() or "en"
+ from openviking.session.memory.utils.language import resolve_output_language
# Build file index mapping and summary string
file_index_map = {}
@@ -1091,7 +1088,7 @@ class SemanticProcessor(DequeueHandlerBase):
file_summaries_lines.append(f"[{idx}] {item['name']}: {item['summary']}")
file_summaries_str = "\n".join(file_summaries_lines) if file_summaries_lines else "None"
- output_language = _detect_language_from_text(file_summaries_str, fallback_language)
+ output_language = resolve_output_language(file_summaries_str, config=config)
# Build subdirectory summary string
children_abstracts_str = (
diff --git a/openviking_cli/utils/config/open_viking_config.py b/openviking_cli/utils/config/open_viking_config.py
index 8b6a92c89..46d85b35d 100644
--- a/openviking_cli/utils/config/open_viking_config.py
+++ b/openviking_cli/utils/config/open_viking_config.py
@@ -7,7 +7,7 @@ from pathlib import Path
from threading import Lock
from typing import Any, Dict, Optional
-from pydantic import BaseModel, Field, ValidationError
+from pydantic import BaseModel, Field, ValidationError, model_validator
from openviking_cli.session.user_id import UserIdentifier
@@ -126,11 +126,31 @@ class OpenVikingConfig(BaseModel):
language_fallback: str = Field(
default="en",
description=(
- "Fallback language used by memory extraction and semantic processing when dominant "
- "user language cannot be confidently detected"
+ "Deprecated. No longer used — detection falls back to 'en' when no language can be "
+ "inferred. Set output_language_override instead to pin an explicit language."
),
)
+ output_language_override: str = Field(
+ default="",
+ description=(
+ "When non-empty, bypasses content-based language detection for memory extraction "
+ "and semantic summaries/overviews and forces this language instead. Use when your "
+ "corpus is mixed-language but you want summaries pinned to a single language "
+ "(e.g., 'en', 'zh-CN', 'ja'). Leave empty (default) to auto-detect per content."
+ ),
+ )
+
+ @model_validator(mode="after")
+ def _warn_on_deprecated_language_fallback(self) -> "OpenVikingConfig":
+ if self.language_fallback and self.language_fallback != "en":
+ logging.getLogger(__name__).warning(
+ "Config field 'language_fallback=%s' is deprecated and has no effect; "
+ "remove it, or set 'output_language_override' to pin an explicit language.",
+ self.language_fallback,
+ )
+ return self
+
allow_private_networks: bool = Field(
default=False,
description=(
diff --git a/tests/storage/test_semantic_processor_language.py b/tests/storage/test_semantic_processor_language.py
index 18e60bd2f..dc441d472 100644
--- a/tests/storage/test_semantic_processor_language.py
+++ b/tests/storage/test_semantic_processor_language.py
@@ -10,7 +10,11 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from openviking.prompts import render_prompt
-from openviking.session.memory.utils.language import _detect_language_from_text
+from openviking.session.memory.utils.language import (
+ _detect_language_from_text,
+ resolve_output_language,
+ resolve_output_language_from_conversation,
+)
class TestLanguageDetection:
@@ -317,3 +321,52 @@ class TestGenerateTextSummaryOutputLanguage:
assert _verify_content_language(result["summary"], expected_lang), (
f"{file_name}: Content language mismatch. Expected {expected_lang}, got: {result['summary']}"
)
+
+
+class TestOutputLanguageOverride:
+ """Config-level `output_language_override` bypasses content-based detection."""
+
+ def _make_config(self, override: str = "", fallback: str = "en"):
+ config = MagicMock()
+ config.output_language_override = override
+ config.language_fallback = fallback
+ return config
+
+ def test_override_unset_detects_from_content(self):
+ config = self._make_config(override="")
+ result = resolve_output_language("これは日本語のテキストです", config=config)
+ assert result == "ja"
+
+ def test_override_unset_uses_fallback_for_latin_text(self):
+ config = self._make_config(override="", fallback="en")
+ result = resolve_output_language(
+ "Plain English text with no special scripts", config=config
+ )
+ assert result == "en"
+
+ def test_override_set_bypasses_detection(self):
+ config = self._make_config(override="en")
+ result = resolve_output_language("これは日本語のテキストです", config=config)
+ assert result == "en"
+
+ def test_override_set_wins_over_fallback(self):
+ config = self._make_config(override="zh-CN", fallback="en")
+ result = resolve_output_language("Plain English text", config=config)
+ assert result == "zh-CN"
+
+ def test_override_whitespace_treated_as_unset(self):
+ config = self._make_config(override=" ")
+ result = resolve_output_language("これは日本語のテキストです", config=config)
+ assert result == "ja"
+
+ def test_conversation_override_set_bypasses_detection(self):
+ config = self._make_config(override="en")
+ conversation = "[user]: これは日本語のメッセージです\n[assistant]: reply"
+ result = resolve_output_language_from_conversation(conversation, config=config)
+ assert result == "en"
+
+ def test_conversation_override_unset_detects_from_user_content(self):
+ config = self._make_config(override="")
+ conversation = "[user]: これは日本語のメッセージです\n[assistant]: reply"
+ result = resolve_output_language_from_conversation(conversation, config=config)
+ assert result == "ja"