Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(embedding): 限制队列输入并停止超长重试 (#2197)
Qin Haojie2f12439b61cf
diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.pyindex 33d02d3ca..15c260362 100644--- a/openviking/storage/collection_schemas.py+++ b/openviking/storage/collection_schemas.py@@ -33,7 +33,8 @@ from openviking.utils.circuit_breaker import (CircuitBreakerOpen,classify_api_error,)-from openviking.utils.model_retry import ERROR_CLASS_PERMANENT+from openviking.utils.embedding_input import truncate_embedding_input+from openviking.utils.model_retry import ERROR_CLASS_INPUT_TOO_LARGE, ERROR_CLASS_PERMANENTfrom openviking_cli.session.user_id import UserIdentifierfrom openviking_cli.utils import get_loggerfrom openviking_cli.utils.config.open_viking_config import OpenVikingConfig@@ -322,6 +323,7 @@ class TextEmbeddingHandler(DequeueHandlerBase):config = get_openviking_config()self._collection_name = config.storage.vectordb.nameself._vector_dim = config.embedding.dimension+ self._max_input_tokens = int(getattr(config.embedding, "max_input_tokens", 4096) or 4096)self._initialize_embedder(config)breaker_cfg = config.embedding.circuit_breakerself._circuit_breaker = CircuitBreaker(@@ -498,8 +500,12 @@ class TextEmbeddingHandler(DequeueHandlerBase):import time as _time_embed_t0 = _time.monotonic()+ embedding_input = truncate_embedding_input(+ embedding_msg.message,+ self._max_input_tokens,+ )result: EmbedResult = await embed_compat(- self._embedder, embedding_msg.message, is_query=False+ self._embedder, embedding_input, is_query=False)_embed_elapsed = _time.monotonic() - _embed_t0try:@@ -527,6 +533,13 @@ class TextEmbeddingHandler(DequeueHandlerBase):except Exception:pass+ if error_class == ERROR_CLASS_INPUT_TOO_LARGE:+ logger.error(error_msg)+ self._merge_request_stats(embedding_msg.telemetry_id, error_count=1)+ request_failed_message = error_msg+ report_error_args = (error_msg, data)+ return None+if error_class == ERROR_CLASS_PERMANENT:logger.critical(error_msg)self._circuit_breaker.record_failure(embed_err)diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex 4389c96d6..d47485588 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -43,7 +43,7 @@ from openviking.utils.circuit_breaker import (CircuitBreakerOpen,classify_api_error,)-from openviking.utils.model_retry import ERROR_CLASS_PERMANENT+from openviking.utils.model_retry import ERROR_CLASS_INPUT_TOO_LARGE, ERROR_CLASS_PERMANENTfrom openviking_cli.session.user_id import UserIdentifierfrom openviking_cli.utils import VikingURIfrom openviking_cli.utils.config import get_openviking_config@@ -423,7 +423,18 @@ class SemanticProcessor(DequeueHandlerBase):except Exception as e:error_class = classify_api_error(e)- if error_class == ERROR_CLASS_PERMANENT:+ if error_class == ERROR_CLASS_INPUT_TOO_LARGE:+ logger.error(+ f"Input too large processing semantic message, dropping: {e}",+ exc_info=True,+ )+ if msg is not None:+ self._merge_request_stats(msg.telemetry_id, error_count=1)+ get_request_wait_tracker().mark_semantic_failed(+ msg.telemetry_id, msg.id, str(e)+ )+ self.report_error(str(e), data)+ elif error_class == ERROR_CLASS_PERMANENT:logger.critical(f"Permanent API error processing semantic message, dropping: {e}",exc_info=True,diff --git a/openviking/utils/circuit_breaker.py b/openviking/utils/circuit_breaker.pyindex c8a67d9aa..726f78d12 100644--- a/openviking/utils/circuit_breaker.py+++ b/openviking/utils/circuit_breaker.py@@ -8,6 +8,7 @@ import threadingimport timefrom openviking.utils.model_retry import (+ ERROR_CLASS_INPUT_TOO_LARGE,ERROR_CLASS_PERMANENT,ERROR_CLASS_QUOTA_EXCEEDED,classify_api_error,@@ -94,6 +95,10 @@ class CircuitBreaker:def record_failure(self, error: Exception) -> None:"""Record a failed API call. May trip the breaker."""error_class = classify_api_error(error)+ if error_class == ERROR_CLASS_INPUT_TOO_LARGE:+ logger.info(f"Circuit breaker ignoring row-specific input error: {error}")+ return+with self._lock:self._failure_count += 1self._last_failure_time = time.monotonic()diff --git a/openviking/utils/embedding_input.py b/openviking/utils/embedding_input.pynew file mode 100644index 000000000..1eb35bd8b--- /dev/null+++ b/openviking/utils/embedding_input.py@@ -0,0 +1,48 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Helpers for bounding text sent to embedding providers."""++from __future__ import annotations++import math++EMBEDDING_TRUNCATION_SUFFIX = "\n...(truncated for embedding)"+++def estimate_embedding_input_tokens(text: str) -> int:+ """Estimate tokens for the raw text embedding input guard."""+ if not text:+ return 0+ cjk_chars = sum(+ 1+ for char in text+ if "\u4e00" <= char <= "\u9fff"+ or "\u3040" <= char <= "\u30ff"+ or "\uac00" <= char <= "\ud7af"+ )+ other_chars = len(text) - cjk_chars+ return max(1, cjk_chars + math.ceil(other_chars / 4))+++def truncate_embedding_input(+ text: str,+ max_tokens: int,+ suffix: str = EMBEDDING_TRUNCATION_SUFFIX,+) -> str:+ """Trim raw text before embedding using the local estimate above."""+ if not text:+ return text+ if max_tokens <= 0:+ return suffix.lstrip()+ if estimate_embedding_input_tokens(text) <= max_tokens:+ return text++ low = 0+ high = len(text)+ while low < high:+ mid = (low + high + 1) // 2+ if estimate_embedding_input_tokens(text[:mid]) <= max_tokens:+ low = mid+ else:+ high = mid - 1+ return text[:low].rstrip() + suffixdiff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.pyindex 6e87a15af..af28cdfda 100644--- a/openviking/utils/embedding_utils.py+++ b/openviking/utils/embedding_utils.py@@ -6,7 +6,6 @@ Embedding utilities for OpenViking.Common logic for creating Context objects and enqueuing them to EmbeddingQueue."""-import mathimport osfrom datetime import datetime, timezonefrom typing import Any, Dict, Optional@@ -17,12 +16,12 @@ from openviking.server.identity import RequestContextfrom openviking.storage.queuefs import get_queue_managerfrom openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverterfrom openviking.storage.viking_fs import get_viking_fs+from openviking.utils.embedding_input import truncate_embedding_inputfrom openviking.utils.time_utils import parse_iso_datetimefrom openviking_cli.utils import VikingURI, get_loggerfrom openviking_cli.utils.config import get_openviking_configlogger = get_logger(__name__)-_EMBEDDING_TRUNCATION_SUFFIX = "\n...(truncated for embedding)"_PORTABLE_SCALAR_FIELDS = frozenset({"type",@@ -44,45 +43,6 @@ def _apply_scalar_overrides(embedding_msg, overrides: Optional[Dict[str, Any]])embedding_msg.context_data[field] = value-def _estimate_embedding_input_tokens(text: str) -> int:- """Estimate tokens for the raw text embedding input guard."""- if not text:- return 0- cjk_chars = sum(- 1- for char in text- if "\u4e00" <= char <= "\u9fff"- or "\u3040" <= char <= "\u30ff"- or "\uac00" <= char <= "\ud7af"- )- other_chars = len(text) - cjk_chars- return max(1, cjk_chars + math.ceil(other_chars / 4))---def _truncate_embedding_input(- text: str,- max_tokens: int,- suffix: str = _EMBEDDING_TRUNCATION_SUFFIX,-) -> str:- """Trim raw text before embedding using the local estimate above."""- if not text:- return text- if max_tokens <= 0:- return suffix.lstrip()- if _estimate_embedding_input_tokens(text) <= max_tokens:- return text-- low = 0- high = len(text)- while low < high:- mid = (low + high + 1) // 2- if _estimate_embedding_input_tokens(text[:mid]) <= max_tokens:- low = mid- else:- high = mid - 1- return text[:low].rstrip() + suffix--async def _decrement_embedding_tracker(semantic_msg_id: Optional[str], count: int) -> None:if not semantic_msg_id or count <= 0:return@@ -417,7 +377,7 @@ async def vectorize_file(content = await viking_fs.read_file(file_path, ctx=ctx)if isinstance(content, bytes):content = content.decode("utf-8", errors="replace")- content = _truncate_embedding_input(content, max_input_tokens)+ content = truncate_embedding_input(content, max_input_tokens)context.set_vectorize(Vectorize(text=content))except Exception as e:logger.warning(diff --git a/openviking/utils/model_retry.py b/openviking/utils/model_retry.pyindex 49a76567e..d90eac8eb 100644--- a/openviking/utils/model_retry.py+++ b/openviking/utils/model_retry.py@@ -13,10 +13,28 @@ T = TypeVar("T")# Error classification categories returned by classify_api_error()ERROR_CLASS_PERMANENT = "permanent"+ERROR_CLASS_INPUT_TOO_LARGE = "input_too_large"ERROR_CLASS_QUOTA_EXCEEDED = "quota_exceeded"ERROR_CLASS_TRANSIENT = "transient"ERROR_CLASS_UNKNOWN = "unknown"+INPUT_TOO_LARGE_PATTERNS = (+ "413",+ "payload too large",+ "request entity too large",+ "content too large",+ "contextwindowexceeded",+ "context window exceeded",+ "maximum context length",+ "max input tokens",+ "too many input tokens",+ "input length exceeds",+ "exceeds the context length",+ "exceeds the max input length",+ "is too large to process",+ "expected maxlength",+)+PERMANENT_API_ERROR_PATTERNS = ("400","401",@@ -27,7 +45,7 @@ PERMANENT_API_ERROR_PATTERNS = ()QUOTA_EXCEEDED_PATTERNS = (- "quotaexceeded", # also 429+ "quotaexceeded", # also 429"quota limit","quota exceed","usage quota",@@ -67,6 +85,13 @@ def classify_api_error(error: Exception) -> str:if error.__cause__ is not None:texts.append(str(error.__cause__))+ for text in texts:+ text_lower = text.lower()+ text_compact = text_lower.replace(" ", "")+ for pattern in INPUT_TOO_LARGE_PATTERNS:+ if pattern in text_lower or pattern in text_compact:+ return ERROR_CLASS_INPUT_TOO_LARGE+for text in texts:text_lower = text.lower()for pattern in PERMANENT_API_ERROR_PATTERNS:@@ -261,9 +286,7 @@ class PrimaryBackupSwitcher:if error_class in (ERROR_CLASS_PERMANENT, ERROR_CLASS_QUOTA_EXCEEDED):with self._lock:if not self._using_backup:- logger.warning(- f"Primary failed with {error_class}, switching to backup"- )+ logger.warning(f"Primary failed with {error_class}, switching to backup")self._using_backup = True# Always reset timer and counter when we fail (whether initial fail or failback fail)self._switch_to_backup_time = time.monotonic()diff --git a/tests/storage/test_collection_schemas.py b/tests/storage/test_collection_schemas.pyindex e071ee153..f87f89817 100644--- a/tests/storage/test_collection_schemas.py+++ b/tests/storage/test_collection_schemas.py@@ -40,6 +40,7 @@ class _DummyConfig:embedder: _DummyEmbedder,backend: str = "volcengine",volcengine_data_api_key: str | None = None,+ max_input_tokens: int = 4096,):self.storage = SimpleNamespace(vectordb=SimpleNamespace(@@ -58,6 +59,7 @@ class _DummyConfig:),sparse=None,hybrid=None,+ max_input_tokens=max_input_tokens,circuit_breaker=SimpleNamespace(failure_threshold=5,reset_timeout=60.0,@@ -394,6 +396,82 @@ async def test_embedding_handler_propagates_account_id_on_error(monkeypatch):assert captured["account_id"] == "acct-embed-error"+async def test_embedding_handler_truncates_queue_input_before_embed(monkeypatch):+ class _CapturingVikingDB:+ is_closing = False++ async def upsert(self, _data, *, ctx):+ return "rec-1"++ class _CapturingEmbedder:+ def __init__(self):+ self.text = None++ def embed(self, text: str, is_query: bool = False) -> EmbedResult:+ del is_query+ self.text = text+ return EmbedResult(dense_vector=[0.1, 0.2])++ embedder = _CapturingEmbedder()+ monkeypatch.setattr(+ "openviking_cli.utils.config.get_openviking_config",+ lambda: _DummyConfig(embedder, max_input_tokens=10),+ )++ handler = TextEmbeddingHandler(_CapturingVikingDB())+ payload = _build_queue_payload()+ queue_data = json.loads(payload["data"])+ queue_data["message"] = " ".join(f"token-{idx}" for idx in range(200))+ payload["data"] = json.dumps(queue_data)++ await handler.on_dequeue(payload)++ assert embedder.text is not None+ assert embedder.text.endswith("...(truncated for embedding)")+ assert "token-199" not in embedder.text+++async def test_embedding_handler_drops_input_too_large_without_requeue(monkeypatch):+ class _QueueingVikingDB:+ is_closing = False+ has_queue_manager = True++ def __init__(self):+ self.enqueued = []++ async def enqueue_embedding_msg(self, msg):+ self.enqueued.append(msg)+ return None++ class _OversizedInputEmbedder:+ def embed(self, text: str, is_query: bool = False) -> EmbedResult:+ del text, is_query+ raise RuntimeError("Malformed input request: expected maxLength: 50000, actual: 75000")++ vikingdb = _QueueingVikingDB()+ monkeypatch.setattr(+ "openviking_cli.utils.config.get_openviking_config",+ lambda: _DummyConfig(_OversizedInputEmbedder()),+ )++ handler = TextEmbeddingHandler(vikingdb)+ status = {"success": 0, "requeue": 0, "error": 0}+ handler.set_callbacks(+ on_success=lambda: status.__setitem__("success", status["success"] + 1),+ on_requeue=lambda: status.__setitem__("requeue", status["requeue"] + 1),+ on_error=lambda *_: status.__setitem__("error", status["error"] + 1),+ )++ result = await handler.on_dequeue(_build_queue_payload())++ assert result is None+ assert vikingdb.enqueued == []+ assert status == {"success": 0, "requeue": 0, "error": 1}+ assert handler._circuit_breaker._failure_count == 0++@pytest.mark.asyncioasync def test_embedding_handler_preserves_parent_uri_for_backend_upsert_logic(monkeypatch):captured = {}diff --git a/tests/unit/test_circuit_breaker.py b/tests/unit/test_circuit_breaker.pyindex f3bf77ccb..28bed02ab 100644--- a/tests/unit/test_circuit_breaker.py+++ b/tests/unit/test_circuit_breaker.py@@ -14,7 +14,6 @@ from openviking.utils.circuit_breaker import ()from openviking.utils.model_retry import (ERROR_CLASS_PERMANENT,- ERROR_CLASS_QUOTA_EXCEEDED,ERROR_CLASS_TRANSIENT,ERROR_CLASS_UNKNOWN,)@@ -166,6 +165,15 @@ class TestCircuitBreaker:with pytest.raises(CircuitBreakerOpen):cb.check()+ def test_input_too_large_does_not_trip_or_increment(self):+ """Test row-specific input errors do not affect global circuit state."""+ cb = CircuitBreaker(failure_threshold=1)++ cb.record_failure(Exception("expected maxLength: 50000, actual: 75000"))++ cb.check()+ assert cb._failure_count == 0+def test_success_resets_failure_count(self):"""Test success resets failure count."""cb = CircuitBreaker(failure_threshold=3)diff --git a/tests/unit/test_model_retry.py b/tests/unit/test_model_retry.pyindex be59be298..996d1e8fe 100644--- a/tests/unit/test_model_retry.py+++ b/tests/unit/test_model_retry.py@@ -5,10 +5,10 @@import pytestfrom openviking.utils.model_retry import (+ ERROR_CLASS_INPUT_TOO_LARGE,ERROR_CLASS_PERMANENT,ERROR_CLASS_QUOTA_EXCEEDED,ERROR_CLASS_TRANSIENT,- ERROR_CLASS_UNKNOWN,classify_api_error,is_quota_exceeded_api_error,retry_async,@@ -125,3 +125,27 @@ def test_quota_exceeded_case_insensitive():"""Quota detection is case-insensitive."""assert classify_api_error(RuntimeError("QUOTA LIMIT")) == ERROR_CLASS_QUOTA_EXCEEDEDassert classify_api_error(RuntimeError("Quota Exceed")) == ERROR_CLASS_QUOTA_EXCEEDED+++ "message",+ [+ "BadRequestError: 400 maximum context length is 8192 tokens",+ "Error code: 413 - Payload Too Large",+ ],+)+def test_classify_input_too_large_errors(message):+ assert classify_api_error(RuntimeError(message)) == ERROR_CLASS_INPUT_TOO_LARGE+++def test_retry_sync_does_not_retry_input_too_large():+ attempts = {"count": 0}++ def _call():+ attempts["count"] += 1+ raise RuntimeError("expected maxLength: 50000, actual: 75000")++ with pytest.raises(RuntimeError, match="expected maxLength"):+ retry_sync(_call, max_retries=5)++ assert attempts["count"] == 1