Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(bot): support OpenAI and credential streaming (#3696)
DuTaod328b054329e
diff --git a/bot/vikingbot/providers/vlm_adapter.py b/bot/vikingbot/providers/vlm_adapter.pyindex 490db3e35..aaa8a09fa 100644--- a/bot/vikingbot/providers/vlm_adapter.py+++ b/bot/vikingbot/providers/vlm_adapter.py@@ -133,13 +133,7 @@ class VLMProviderAdapter(LLMProvider):temperature: float = 0.7,session_id: str | None = None,) -> AsyncIterator[LLMStreamEvent]:- # Failover wrappers expose the primary provider name but intentionally- # do not expose a single provider client. Route them through chat() so- # get_completion_async() can select the active credential safely.- supports_native_volcengine_stream = getattr(- self._vlm, "provider", None- ) == "volcengine" and callable(getattr(self._vlm, "get_async_client", None))- if not supports_native_volcengine_stream:+ if not self._supports_native_stream(self._vlm):async for event in super().chat_stream(messages=messages,tools=tools,@@ -151,131 +145,264 @@ class VLMProviderAdapter(LLMProvider):yield eventreturn- async for event in self._chat_stream_volcengine(- messages=messages,- tools=tools,- model=model,- max_tokens=max_tokens,- temperature=temperature,- ):- yield event+ try:+ stream_with_failover = getattr(self._vlm, "stream_with_failover", None)+ if callable(stream_with_failover):+ async for event in self._chat_stream_failover_with_retry(+ stream_with_failover,+ messages=messages,+ tools=tools,+ model=model,+ max_tokens=max_tokens,+ temperature=temperature,+ ):+ yield event+ return++ async for event in self._chat_stream_with_retry(+ self._vlm,+ messages=messages,+ tools=tools,+ model=model,+ max_tokens=max_tokens,+ temperature=temperature,+ ):+ yield event+ except Exception as exc:+ yield LLMStreamEvent(+ type="response",+ response=LLMResponse(+ content=f"Error calling LLM in VLM Adapter stream: {str(exc)}",+ finish_reason="error",+ ),+ )++ @classmethod+ def _supports_native_stream(cls, vlm: Any) -> bool:+ instances = getattr(vlm, "_vlm_instances", None)+ if instances is not None:+ return bool(instances) and all(cls._supports_native_stream(item) for item in instances)++ primary = getattr(vlm, "primary", None)+ backup = getattr(vlm, "backup", None)+ if primary is not None and backup is not None:+ return cls._supports_native_stream(primary) and cls._supports_native_stream(backup)++ return getattr(vlm, "provider", None) in {"openai", "volcengine"} and callable(+ getattr(vlm, "get_async_client", None)+ )- async def _chat_stream_volcengine(+ async def _chat_stream_failover_with_retry(self,+ stream_with_failover: Any,+ *,messages: list[dict[str, Any]],tools: list[dict[str, Any]] | None,model: str | None,max_tokens: int | None,temperature: float,) -> AsyncIterator[LLMStreamEvent]:- configured_max_tokens = getattr(self._vlm, "max_tokens", None)- effective_max_tokens = (- configured_max_tokens if configured_max_tokens is not None else max_tokens- )- kwargs: dict[str, Any] = {- "model": model or getattr(self._vlm, "model", None) or self._default_model,- "messages": messages,- "temperature": getattr(self._vlm, "temperature", temperature),- "thinking": {- "type": "enabled" if getattr(self._vlm, "thinking", False) else "disabled"- },- "stream": True,- "stream_options": {"include_usage": True},- }- if effective_max_tokens is not None:- kwargs["max_tokens"] = effective_max_tokens- extra_headers = getattr(self._vlm, "extra_headers", None)- if extra_headers:- kwargs["extra_headers"] = extra_headers- if tools:- kwargs["tools"] = tools- kwargs["tool_choice"] = "auto"-- content_parts: list[str] = []- reasoning_parts: list[str] = []- tool_calls: dict[int, dict[str, Any]] = {}- finish_reason = "stop"- usage: dict[str, int] = {}-attempt = 1while True:- content_parts.clear()- reasoning_parts.clear()- tool_calls.clear()- finish_reason = "stop"- usage = {}- start_time = time.perf_counter()+ emitted = Falsetry:- client = self._vlm.get_async_client()- response = await client.chat.completions.create(**kwargs)- async for chunk in response:- chunk_usage = self._parse_usage(getattr(chunk, "usage", None))- if chunk_usage:- usage = chunk_usage-- choices = getattr(chunk, "choices", None) or []- if not choices:- continue- choice = choices[0]- if getattr(choice, "finish_reason", None):- finish_reason = choice.finish_reason or finish_reason- delta = getattr(choice, "delta", None)- if delta is None:- continue-- reasoning_delta = stream_delta_value(delta, "reasoning_content")- if reasoning_delta:- reasoning_parts.append(reasoning_delta)- yield LLMStreamEvent(type="reasoning_delta", content=reasoning_delta)-- content_delta = stream_delta_value(delta, "content")- if content_delta:- content_parts.append(content_delta)- yield LLMStreamEvent(type="content_delta", content=content_delta)-- for fallback_index, delta_tool_call in enumerate(- getattr(delta, "tool_calls", None) or []- ):- merge_stream_tool_call_delta(- tool_calls,- delta_tool_call,- fallback_index=fallback_index,- )-- if usage:- self._record_vlm_usage(usage, time.perf_counter() - start_time)-- yield LLMStreamEvent(- type="response",- response=build_stream_response(- content="".join(content_parts),- reasoning_content="".join(reasoning_parts),- raw_tool_calls=tool_calls,- finish_reason=finish_reason,- usage=usage,- ),+ async for event in stream_with_failover(+ lambda vlm: self._chat_stream_backend(+ vlm,+ messages=messages,+ tools=tools,+ model=model,+ max_tokens=max_tokens,+ temperature=temperature,+ )+ ):+ emitted = True+ yield event+ return+ except Exception as exc:+ if emitted or not is_retryable_rate_limit_error(exc):+ raise+ delay = rate_limit_retry_delay(attempt)+ logger.warning(+ "VLM credential stream rate limited; retrying attempt={} "+ "delay={:.1f}s error={}",+ attempt,+ delay,+ exc,)+ await asyncio.sleep(delay)+ attempt += 1++ async def _chat_stream_with_retry(+ self,+ vlm: Any,+ messages: list[dict[str, Any]],+ tools: list[dict[str, Any]] | None,+ model: str | None,+ max_tokens: int | None,+ temperature: float,+ ) -> AsyncIterator[LLMStreamEvent]:+ attempt = 1+ while True:+ emitted = False+ try:+ async for event in self._chat_stream_backend(+ vlm,+ messages=messages,+ tools=tools,+ model=model,+ max_tokens=max_tokens,+ temperature=temperature,+ ):+ emitted = True+ yield eventreturn- except Exception as e:- if not is_retryable_rate_limit_error(e):- yield LLMStreamEvent(- type="response",- response=LLMResponse(- content=f"Error calling LLM in VLM Adapter stream: {str(e)}",- finish_reason="error",- ),- )- return+ except Exception as exc:+ if emitted or not is_retryable_rate_limit_error(exc):+ raisedelay = rate_limit_retry_delay(attempt)logger.warning("VLM adapter stream rate limited; retrying attempt={} delay={:.1f}s error={}",attempt,delay,- e,+ exc,)await asyncio.sleep(delay)attempt += 1+ async def _chat_stream_backend(+ self,+ vlm: Any,+ *,+ messages: list[dict[str, Any]],+ tools: list[dict[str, Any]] | None,+ model: str | None,+ max_tokens: int | None,+ temperature: float,+ ) -> AsyncIterator[LLMStreamEvent]:+ kwargs = self._build_stream_kwargs(+ vlm,+ messages=messages,+ tools=tools,+ model=model,+ max_tokens=max_tokens,+ temperature=temperature,+ )+ content_parts: list[str] = []+ reasoning_parts: list[str] = []+ tool_calls: dict[int, dict[str, Any]] = {}+ finish_reason = "stop"+ usage: dict[str, int] = {}+ start_time = time.perf_counter()++ client = vlm.get_async_client()+ response = await client.chat.completions.create(**kwargs)+ async for chunk in response:+ chunk_usage = self._parse_usage(getattr(chunk, "usage", None))+ if chunk_usage:+ usage = chunk_usage++ choices = getattr(chunk, "choices", None) or []+ if not choices:+ continue+ choice = choices[0]+ if getattr(choice, "finish_reason", None):+ finish_reason = choice.finish_reason or finish_reason+ delta = getattr(choice, "delta", None)+ if delta is None:+ continue++ reasoning_delta = stream_delta_value(delta, "reasoning_content")+ if reasoning_delta:+ reasoning_parts.append(reasoning_delta)+ yield LLMStreamEvent(type="reasoning_delta", content=reasoning_delta)++ content_delta = stream_delta_value(delta, "content")+ if content_delta:+ content_parts.append(content_delta)+ yield LLMStreamEvent(type="content_delta", content=content_delta)++ for fallback_index, delta_tool_call in enumerate(+ getattr(delta, "tool_calls", None) or []+ ):+ merge_stream_tool_call_delta(+ tool_calls,+ delta_tool_call,+ fallback_index=fallback_index,+ )++ if usage:+ self._record_vlm_usage(vlm, usage, time.perf_counter() - start_time)++ yield LLMStreamEvent(+ type="response",+ response=build_stream_response(+ content="".join(content_parts),+ reasoning_content="".join(reasoning_parts),+ raw_tool_calls=tool_calls,+ finish_reason=finish_reason,+ usage=usage,+ ),+ )++ def _build_stream_kwargs(+ self,+ vlm: Any,+ *,+ messages: list[dict[str, Any]],+ tools: list[dict[str, Any]] | None,+ model: str | None,+ max_tokens: int | None,+ temperature: float,+ ) -> dict[str, Any]:+ provider = getattr(vlm, "provider", None)+ configured_max_tokens = getattr(vlm, "max_tokens", None)+ effective_max_tokens = (+ configured_max_tokens if configured_max_tokens is not None else max_tokens+ )+ # A wrapped credential owns its model. The model passed by AgentLoop is+ # the outer/default model and must not overwrite a backup credential.+ effective_model = (+ getattr(vlm, "model", None)+ if vlm is not self._vlm+ else model or getattr(vlm, "model", None)+ ) or self._default_model++ if provider == "volcengine":+ kwargs: dict[str, Any] = {+ "model": effective_model,+ "messages": messages,+ "temperature": getattr(vlm, "temperature", temperature),+ "thinking": {"type": "enabled" if getattr(vlm, "thinking", False) else "disabled"},+ "stream": True,+ "stream_options": {"include_usage": True},+ }+ if effective_max_tokens is not None:+ kwargs["max_tokens"] = effective_max_tokens+ extra_headers = getattr(vlm, "extra_headers", None)+ if extra_headers:+ kwargs["extra_headers"] = extra_headers+ if tools:+ kwargs["tools"] = tools+ kwargs["tool_choice"] = "auto"+ return kwargs++ kwargs = vlm._build_text_kwargs(+ messages=messages,+ tools=tools,+ tool_choice="auto" if tools else None,+ thinking=getattr(vlm, "thinking", None),+ )+ kwargs["model"] = effective_model+ kwargs["stream"] = True+ kwargs["stream_options"] = {"include_usage": True}+ if effective_max_tokens is not None and not (+ "max_tokens" in kwargs or "max_completion_tokens" in kwargs+ ):+ token_key = "max_completion_tokens" if "reasoning_effort" in kwargs else "max_tokens"+ kwargs[token_key] = effective_max_tokens+ return kwargs+@staticmethoddef _usage_value(usage: Any, name: str) -> int:if isinstance(usage, dict):@@ -312,14 +439,19 @@ class VLMProviderAdapter(LLMProvider):usage["reasoning_tokens"] = reasoningreturn usage- def _record_vlm_usage(self, usage: dict[str, int], duration_seconds: float) -> None:- update_token_usage = getattr(self._vlm, "update_token_usage", None)+ def _record_vlm_usage(+ self,+ vlm: Any,+ usage: dict[str, int],+ duration_seconds: float,+ ) -> None:+ update_token_usage = getattr(vlm, "update_token_usage", None)if not callable(update_token_usage):returntry:update_token_usage(- model_name=getattr(self._vlm, "model", None) or self._default_model,- provider=getattr(self._vlm, "provider", None) or "volcengine",+ model_name=getattr(vlm, "model", None) or self._default_model,+ provider=getattr(vlm, "provider", None) or "unknown",prompt_tokens=usage.get("prompt_tokens", 0),completion_tokens=usage.get("completion_tokens", 0),duration_seconds=duration_seconds,diff --git a/bot/vikingbot/tests/unit/test_bot_vlm_streaming.py b/bot/vikingbot/tests/unit/test_bot_vlm_streaming.pynew file mode 100644index 000000000..70697d30e--- /dev/null+++ b/bot/vikingbot/tests/unit/test_bot_vlm_streaming.py@@ -0,0 +1,205 @@+from types import SimpleNamespace++import pytest++from openviking.models.vlm import FailoverVLM, MultiCredentialVLM, OpenAIVLM, VolcEngineVLM+from vikingbot.providers.vlm_adapter import VLMProviderAdapter+++class FakeCompletions:+ def __init__(self, *, chunks=None, error: Exception | None = None):+ self.chunks = chunks or []+ self.error = error+ self.calls: list[dict] = []++ async def create(self, **kwargs):+ self.calls.append(kwargs)+ if self.error is not None:+ raise self.error++ async def generate():+ for chunk in self.chunks:+ if isinstance(chunk, Exception):+ raise chunk+ yield chunk++ return generate()+++def stream_chunk(+ content: str | None = None,+ *,+ finish_reason: str | None = None,+ usage=None,+):+ choices = []+ if content is not None or finish_reason is not None:+ choices.append(+ SimpleNamespace(+ finish_reason=finish_reason,+ delta=SimpleNamespace(+ reasoning_content=None,+ content=content,+ tool_calls=[],+ ),+ )+ )+ return SimpleNamespace(choices=choices, usage=usage)+++def attach_client(vlm, completions: FakeCompletions) -> None:+ vlm.get_async_client = lambda: SimpleNamespace(chat=SimpleNamespace(completions=completions))+++def make_adapter(vlm, default_model: str) -> VLMProviderAdapter:+ return VLMProviderAdapter(+ vlm,+ default_model=default_model,+ langfuse_client=SimpleNamespace(enabled=False, _client=None),+ )+++async def test_vlm_adapter_streams_openai_content_deltas():+ usage = SimpleNamespace(+ prompt_tokens=3,+ completion_tokens=2,+ total_tokens=5,+ prompt_tokens_details=None,+ completion_tokens_details=None,+ )+ completions = FakeCompletions(+ chunks=[+ stream_chunk("hel"),+ stream_chunk("lo", finish_reason="stop"),+ stream_chunk(usage=usage),+ ]+ )+ vlm = OpenAIVLM(+ {+ "provider": "openai",+ "model": "gpt-test",+ "api_key": "test-key",+ "temperature": 0.1,+ }+ )+ attach_client(vlm, completions)+ provider = make_adapter(vlm, "gpt-test")++ events = [+ event+ async for event in provider.chat_stream(+ messages=[{"role": "user", "content": "hi"}],+ )+ ]++ assert [event.type for event in events] == [+ "content_delta",+ "content_delta",+ "response",+ ]+ assert [event.content for event in events[:-1]] == ["hel", "lo"]+ assert events[-1].response.content == "hello"+ assert events[-1].response.usage == {+ "prompt_tokens": 3,+ "completion_tokens": 2,+ "total_tokens": 5,+ }+ assert completions.calls[0]["stream"] is True+ assert completions.calls[0]["stream_options"] == {"include_usage": True}+ assert "thinking" not in completions.calls[0]+++async def test_vlm_adapter_streams_with_credential_failover_before_first_delta():+ primary_completions = FakeCompletions(error=RuntimeError("status 429: rate limit"))+ backup_completions = FakeCompletions(chunks=[stream_chunk("backup", finish_reason="stop")])+ primary = VolcEngineVLM(+ {+ "provider": "volcengine",+ "model": "primary-model",+ "api_key": "primary-key",+ }+ )+ backup = VolcEngineVLM(+ {+ "provider": "volcengine",+ "model": "backup-model",+ "api_key": "backup-key",+ }+ )+ attach_client(primary, primary_completions)+ attach_client(backup, backup_completions)+ credentials = MultiCredentialVLM(+ [primary, backup],+ credential_ids=["primary", "backup"],+ )+ provider = make_adapter(credentials, "primary-model")++ events = [+ event+ async for event in provider.chat_stream(+ messages=[{"role": "user", "content": "hi"}],+ model="primary-model",+ )+ ]++ assert [event.type for event in events] == ["content_delta", "response"]+ assert events[0].content == "backup"+ assert events[-1].response.content == "backup"+ assert len(primary_completions.calls) == 1+ assert backup_completions.calls[0]["model"] == "backup-model"+ assert credentials.active_credential_id == "backup"+++async def test_vlm_adapter_streams_with_legacy_primary_backup_failover():+ primary_completions = FakeCompletions(error=RuntimeError("status 401: unauthorized"))+ backup_completions = FakeCompletions(chunks=[stream_chunk("backup", finish_reason="stop")])+ primary = OpenAIVLM({"provider": "openai", "model": "primary-model", "api_key": "primary-key"})+ backup = OpenAIVLM({"provider": "openai", "model": "backup-model", "api_key": "backup-key"})+ attach_client(primary, primary_completions)+ attach_client(backup, backup_completions)+ failover = FailoverVLM(primary, backup)+ provider = make_adapter(failover, "primary-model")++ events = [+ event+ async for event in provider.chat_stream(+ messages=[{"role": "user", "content": "hi"}],+ )+ ]++ assert [event.type for event in events] == ["content_delta", "response"]+ assert events[-1].response.content == "backup"+ assert failover.is_using_backup is True+++async def test_credential_stream_does_not_replay_after_a_visible_delta():+ primary_completions = FakeCompletions(+ chunks=[stream_chunk("partial"), RuntimeError("status 429: rate limit")]+ )+ backup_completions = FakeCompletions(chunks=[stream_chunk("duplicate", finish_reason="stop")])+ primary = OpenAIVLM({"provider": "openai", "model": "primary-model", "api_key": "primary-key"})+ backup = OpenAIVLM({"provider": "openai", "model": "backup-model", "api_key": "backup-key"})+ attach_client(primary, primary_completions)+ attach_client(backup, backup_completions)+ credentials = MultiCredentialVLM(+ [primary, backup],+ credential_ids=["primary", "backup"],+ )+ provider = make_adapter(credentials, "primary-model")++ events = [+ event+ async for event in provider.chat_stream(+ messages=[{"role": "user", "content": "hi"}],+ )+ ]++ assert events[0].type == "content_delta"+ assert events[0].content == "partial"+ assert events[-1].type == "response"+ assert events[-1].response.finish_reason == "error"+ assert backup_completions.calls == []diff --git a/openviking/models/vlm/base.py b/openviking/models/vlm/base.pyindex 640e07e5c..742ac34ad 100644--- a/openviking/models/vlm/base.py+++ b/openviking/models/vlm/base.py@@ -7,7 +7,7 @@ import refrom abc import ABC, abstractmethodfrom dataclasses import dataclass, fieldfrom pathlib import Path-from typing import Any, Dict, List, Optional, Union+from typing import Any, AsyncIterator, Callable, Dict, List, Optional, Unionfrom openviking.utils.exceptions import AllCredentialsFailedErrorfrom openviking.utils.model_retry import (@@ -479,6 +479,45 @@ class FailoverVLM(VLMBase):self._logger.error(f"Backup VLM also failed with error: {e}")raise last_error+ async def stream_with_failover(+ self,+ stream_factory: Callable[[VLMBase], AsyncIterator[Any]],+ ) -> AsyncIterator[Any]:+ """Stream from the active backend and fail over before output starts.++ Once an item has been yielded, replaying the request through another+ backend would duplicate visible output. Errors after that point are+ therefore propagated to the caller instead of triggering failover.+ """+ last_error = None++ if self._switcher.should_try_primary():+ emitted = False+ try:+ async for item in stream_factory(self.primary):+ emitted = True+ yield item+ self._switcher.record_primary_success()+ return+ except Exception as exc:+ _annotate_vlm_error(exc, self.primary)+ last_error = exc+ if emitted or not self._switcher.record_primary_failure(exc):+ raise++ emitted = False+ try:+ self._switcher.record_backup_request()+ async for item in stream_factory(self.backup):+ emitted = True+ yield item+ return+ except Exception as exc:+ _annotate_vlm_error(exc, self.backup)+ last_error = exc+ self._logger.error(f"Backup VLM also failed with error: {exc}")+ raise last_error+def get_completion(self,prompt: str = "",@@ -743,6 +782,48 @@ class MultiCredentialVLM(VLMBase):raise AllCredentialsFailedError(aggregated_errors)+ async def stream_with_failover(+ self,+ stream_factory: Callable[[VLMBase], AsyncIterator[Any]],+ ) -> AsyncIterator[Any]:+ """Stream with ordered credential failover before output starts.++ A credential may be replaced while opening or awaiting the first+ visible stream item. After an item is emitted, switching would replay+ already delivered output, so subsequent errors fail the request.+ """+ aggregated_errors = []+ start = self._switcher.maybe_failback()+ n = self._switcher.n++ for offset in range(n):+ idx = (start + offset) % n+ credential_id = self._credential_ids[idx]+ vlm_instance = self._vlm_instances[idx]+ emitted = False++ try:+ async for item in stream_factory(vlm_instance):+ emitted = True+ yield item+ self._switcher.commit_success(idx)+ return+ except Exception as exc:+ _annotate_vlm_error(exc, vlm_instance)+ if emitted:+ raise++ error_class = classify_api_error(exc)+ aggregated_errors.append((credential_id, error_class, exc, idx))+ if self._switcher.is_fail_fast(error_class):+ raise++ self._logger.warning(+ f"Credential {credential_id} failed with {error_class}, trying next credential"+ )++ raise AllCredentialsFailedError(aggregated_errors)+def get_completion(self,prompt: str = "",