Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(bot): support OpenAI and credential streaming (#3696)

DuTaod328b054329e
diff --git a/bot/vikingbot/providers/vlm_adapter.py b/bot/vikingbot/providers/vlm_adapter.py
index 490db3e35..aaa8a09fa 100644
--- a/bot/vikingbot/providers/vlm_adapter.py
+++ b/bot/vikingbot/providers/vlm_adapter.py
@@ -133,13 +133,7 @@ class VLMProviderAdapter(LLMProvider):
temperature: float = 0.7,
session_id: str | None = None,
) -> AsyncIterator[LLMStreamEvent]:
- # Failover wrappers expose the primary provider name but intentionally
- # do not expose a single provider client. Route them through chat() so
- # get_completion_async() can select the active credential safely.
- supports_native_volcengine_stream = getattr(
- self._vlm, "provider", None
- ) == "volcengine" and callable(getattr(self._vlm, "get_async_client", None))
- if not supports_native_volcengine_stream:
+ if not self._supports_native_stream(self._vlm):
async for event in super().chat_stream(
messages=messages,
tools=tools,
@@ -151,131 +145,264 @@ class VLMProviderAdapter(LLMProvider):
yield event
return
- async for event in self._chat_stream_volcengine(
- messages=messages,
- tools=tools,
- model=model,
- max_tokens=max_tokens,
- temperature=temperature,
- ):
- yield event
+ try:
+ stream_with_failover = getattr(self._vlm, "stream_with_failover", None)
+ if callable(stream_with_failover):
+ async for event in self._chat_stream_failover_with_retry(
+ stream_with_failover,
+ messages=messages,
+ tools=tools,
+ model=model,
+ max_tokens=max_tokens,
+ temperature=temperature,
+ ):
+ yield event
+ return
+
+ async for event in self._chat_stream_with_retry(
+ self._vlm,
+ messages=messages,
+ tools=tools,
+ model=model,
+ max_tokens=max_tokens,
+ temperature=temperature,
+ ):
+ yield event
+ except Exception as exc:
+ yield LLMStreamEvent(
+ type="response",
+ response=LLMResponse(
+ content=f"Error calling LLM in VLM Adapter stream: {str(exc)}",
+ finish_reason="error",
+ ),
+ )
+
+ @classmethod
+ def _supports_native_stream(cls, vlm: Any) -> bool:
+ instances = getattr(vlm, "_vlm_instances", None)
+ if instances is not None:
+ return bool(instances) and all(cls._supports_native_stream(item) for item in instances)
+
+ primary = getattr(vlm, "primary", None)
+ backup = getattr(vlm, "backup", None)
+ if primary is not None and backup is not None:
+ return cls._supports_native_stream(primary) and cls._supports_native_stream(backup)
+
+ return getattr(vlm, "provider", None) in {"openai", "volcengine"} and callable(
+ getattr(vlm, "get_async_client", None)
+ )
- async def _chat_stream_volcengine(
+ async def _chat_stream_failover_with_retry(
self,
+ stream_with_failover: Any,
+ *,
messages: list[dict[str, Any]],
tools: list[dict[str, Any]] | None,
model: str | None,
max_tokens: int | None,
temperature: float,
) -> AsyncIterator[LLMStreamEvent]:
- configured_max_tokens = getattr(self._vlm, "max_tokens", None)
- effective_max_tokens = (
- configured_max_tokens if configured_max_tokens is not None else max_tokens
- )
- kwargs: dict[str, Any] = {
- "model": model or getattr(self._vlm, "model", None) or self._default_model,
- "messages": messages,
- "temperature": getattr(self._vlm, "temperature", temperature),
- "thinking": {
- "type": "enabled" if getattr(self._vlm, "thinking", False) else "disabled"
- },
- "stream": True,
- "stream_options": {"include_usage": True},
- }
- if effective_max_tokens is not None:
- kwargs["max_tokens"] = effective_max_tokens
- extra_headers = getattr(self._vlm, "extra_headers", None)
- if extra_headers:
- kwargs["extra_headers"] = extra_headers
- if tools:
- kwargs["tools"] = tools
- kwargs["tool_choice"] = "auto"
-
- content_parts: list[str] = []
- reasoning_parts: list[str] = []
- tool_calls: dict[int, dict[str, Any]] = {}
- finish_reason = "stop"
- usage: dict[str, int] = {}
-
attempt = 1
while True:
- content_parts.clear()
- reasoning_parts.clear()
- tool_calls.clear()
- finish_reason = "stop"
- usage = {}
- start_time = time.perf_counter()
+ emitted = False
try:
- client = self._vlm.get_async_client()
- response = await client.chat.completions.create(**kwargs)
- async for chunk in response:
- chunk_usage = self._parse_usage(getattr(chunk, "usage", None))
- if chunk_usage:
- usage = chunk_usage
-
- choices = getattr(chunk, "choices", None) or []
- if not choices:
- continue
- choice = choices[0]
- if getattr(choice, "finish_reason", None):
- finish_reason = choice.finish_reason or finish_reason
- delta = getattr(choice, "delta", None)
- if delta is None:
- continue
-
- reasoning_delta = stream_delta_value(delta, "reasoning_content")
- if reasoning_delta:
- reasoning_parts.append(reasoning_delta)
- yield LLMStreamEvent(type="reasoning_delta", content=reasoning_delta)
-
- content_delta = stream_delta_value(delta, "content")
- if content_delta:
- content_parts.append(content_delta)
- yield LLMStreamEvent(type="content_delta", content=content_delta)
-
- for fallback_index, delta_tool_call in enumerate(
- getattr(delta, "tool_calls", None) or []
- ):
- merge_stream_tool_call_delta(
- tool_calls,
- delta_tool_call,
- fallback_index=fallback_index,
- )
-
- if usage:
- self._record_vlm_usage(usage, time.perf_counter() - start_time)
-
- yield LLMStreamEvent(
- type="response",
- response=build_stream_response(
- content="".join(content_parts),
- reasoning_content="".join(reasoning_parts),
- raw_tool_calls=tool_calls,
- finish_reason=finish_reason,
- usage=usage,
- ),
+ async for event in stream_with_failover(
+ lambda vlm: self._chat_stream_backend(
+ vlm,
+ messages=messages,
+ tools=tools,
+ model=model,
+ max_tokens=max_tokens,
+ temperature=temperature,
+ )
+ ):
+ emitted = True
+ yield event
+ return
+ except Exception as exc:
+ if emitted or not is_retryable_rate_limit_error(exc):
+ raise
+ delay = rate_limit_retry_delay(attempt)
+ logger.warning(
+ "VLM credential stream rate limited; retrying attempt={} "
+ "delay={:.1f}s error={}",
+ attempt,
+ delay,
+ exc,
)
+ await asyncio.sleep(delay)
+ attempt += 1
+
+ async def _chat_stream_with_retry(
+ self,
+ vlm: Any,
+ messages: list[dict[str, Any]],
+ tools: list[dict[str, Any]] | None,
+ model: str | None,
+ max_tokens: int | None,
+ temperature: float,
+ ) -> AsyncIterator[LLMStreamEvent]:
+ attempt = 1
+ while True:
+ emitted = False
+ try:
+ async for event in self._chat_stream_backend(
+ vlm,
+ messages=messages,
+ tools=tools,
+ model=model,
+ max_tokens=max_tokens,
+ temperature=temperature,
+ ):
+ emitted = True
+ yield event
return
- except Exception as e:
- if not is_retryable_rate_limit_error(e):
- yield LLMStreamEvent(
- type="response",
- response=LLMResponse(
- content=f"Error calling LLM in VLM Adapter stream: {str(e)}",
- finish_reason="error",
- ),
- )
- return
+ except Exception as exc:
+ if emitted or not is_retryable_rate_limit_error(exc):
+ raise
delay = rate_limit_retry_delay(attempt)
logger.warning(
"VLM adapter stream rate limited; retrying attempt={} delay={:.1f}s error={}",
attempt,
delay,
- e,
+ exc,
)
await asyncio.sleep(delay)
attempt += 1
+ async def _chat_stream_backend(
+ self,
+ vlm: Any,
+ *,
+ messages: list[dict[str, Any]],
+ tools: list[dict[str, Any]] | None,
+ model: str | None,
+ max_tokens: int | None,
+ temperature: float,
+ ) -> AsyncIterator[LLMStreamEvent]:
+ kwargs = self._build_stream_kwargs(
+ vlm,
+ messages=messages,
+ tools=tools,
+ model=model,
+ max_tokens=max_tokens,
+ temperature=temperature,
+ )
+ content_parts: list[str] = []
+ reasoning_parts: list[str] = []
+ tool_calls: dict[int, dict[str, Any]] = {}
+ finish_reason = "stop"
+ usage: dict[str, int] = {}
+ start_time = time.perf_counter()
+
+ client = vlm.get_async_client()
+ response = await client.chat.completions.create(**kwargs)
+ async for chunk in response:
+ chunk_usage = self._parse_usage(getattr(chunk, "usage", None))
+ if chunk_usage:
+ usage = chunk_usage
+
+ choices = getattr(chunk, "choices", None) or []
+ if not choices:
+ continue
+ choice = choices[0]
+ if getattr(choice, "finish_reason", None):
+ finish_reason = choice.finish_reason or finish_reason
+ delta = getattr(choice, "delta", None)
+ if delta is None:
+ continue
+
+ reasoning_delta = stream_delta_value(delta, "reasoning_content")
+ if reasoning_delta:
+ reasoning_parts.append(reasoning_delta)
+ yield LLMStreamEvent(type="reasoning_delta", content=reasoning_delta)
+
+ content_delta = stream_delta_value(delta, "content")
+ if content_delta:
+ content_parts.append(content_delta)
+ yield LLMStreamEvent(type="content_delta", content=content_delta)
+
+ for fallback_index, delta_tool_call in enumerate(
+ getattr(delta, "tool_calls", None) or []
+ ):
+ merge_stream_tool_call_delta(
+ tool_calls,
+ delta_tool_call,
+ fallback_index=fallback_index,
+ )
+
+ if usage:
+ self._record_vlm_usage(vlm, usage, time.perf_counter() - start_time)
+
+ yield LLMStreamEvent(
+ type="response",
+ response=build_stream_response(
+ content="".join(content_parts),
+ reasoning_content="".join(reasoning_parts),
+ raw_tool_calls=tool_calls,
+ finish_reason=finish_reason,
+ usage=usage,
+ ),
+ )
+
+ def _build_stream_kwargs(
+ self,
+ vlm: Any,
+ *,
+ messages: list[dict[str, Any]],
+ tools: list[dict[str, Any]] | None,
+ model: str | None,
+ max_tokens: int | None,
+ temperature: float,
+ ) -> dict[str, Any]:
+ provider = getattr(vlm, "provider", None)
+ configured_max_tokens = getattr(vlm, "max_tokens", None)
+ effective_max_tokens = (
+ configured_max_tokens if configured_max_tokens is not None else max_tokens
+ )
+ # A wrapped credential owns its model. The model passed by AgentLoop is
+ # the outer/default model and must not overwrite a backup credential.
+ effective_model = (
+ getattr(vlm, "model", None)
+ if vlm is not self._vlm
+ else model or getattr(vlm, "model", None)
+ ) or self._default_model
+
+ if provider == "volcengine":
+ kwargs: dict[str, Any] = {
+ "model": effective_model,
+ "messages": messages,
+ "temperature": getattr(vlm, "temperature", temperature),
+ "thinking": {"type": "enabled" if getattr(vlm, "thinking", False) else "disabled"},
+ "stream": True,
+ "stream_options": {"include_usage": True},
+ }
+ if effective_max_tokens is not None:
+ kwargs["max_tokens"] = effective_max_tokens
+ extra_headers = getattr(vlm, "extra_headers", None)
+ if extra_headers:
+ kwargs["extra_headers"] = extra_headers
+ if tools:
+ kwargs["tools"] = tools
+ kwargs["tool_choice"] = "auto"
+ return kwargs
+
+ kwargs = vlm._build_text_kwargs(
+ messages=messages,
+ tools=tools,
+ tool_choice="auto" if tools else None,
+ thinking=getattr(vlm, "thinking", None),
+ )
+ kwargs["model"] = effective_model
+ kwargs["stream"] = True
+ kwargs["stream_options"] = {"include_usage": True}
+ if effective_max_tokens is not None and not (
+ "max_tokens" in kwargs or "max_completion_tokens" in kwargs
+ ):
+ token_key = "max_completion_tokens" if "reasoning_effort" in kwargs else "max_tokens"
+ kwargs[token_key] = effective_max_tokens
+ return kwargs
+
@staticmethod
def _usage_value(usage: Any, name: str) -> int:
if isinstance(usage, dict):
@@ -312,14 +439,19 @@ class VLMProviderAdapter(LLMProvider):
usage["reasoning_tokens"] = reasoning
return usage
- def _record_vlm_usage(self, usage: dict[str, int], duration_seconds: float) -> None:
- update_token_usage = getattr(self._vlm, "update_token_usage", None)
+ def _record_vlm_usage(
+ self,
+ vlm: Any,
+ usage: dict[str, int],
+ duration_seconds: float,
+ ) -> None:
+ update_token_usage = getattr(vlm, "update_token_usage", None)
if not callable(update_token_usage):
return
try:
update_token_usage(
- model_name=getattr(self._vlm, "model", None) or self._default_model,
- provider=getattr(self._vlm, "provider", None) or "volcengine",
+ model_name=getattr(vlm, "model", None) or self._default_model,
+ provider=getattr(vlm, "provider", None) or "unknown",
prompt_tokens=usage.get("prompt_tokens", 0),
completion_tokens=usage.get("completion_tokens", 0),
duration_seconds=duration_seconds,
diff --git a/bot/vikingbot/tests/unit/test_bot_vlm_streaming.py b/bot/vikingbot/tests/unit/test_bot_vlm_streaming.py
new file mode 100644
index 000000000..70697d30e
--- /dev/null
+++ b/bot/vikingbot/tests/unit/test_bot_vlm_streaming.py
@@ -0,0 +1,205 @@
+from types import SimpleNamespace
+
+import pytest
+
+from openviking.models.vlm import FailoverVLM, MultiCredentialVLM, OpenAIVLM, VolcEngineVLM
+from vikingbot.providers.vlm_adapter import VLMProviderAdapter
+
+
+class FakeCompletions:
+ def __init__(self, *, chunks=None, error: Exception | None = None):
+ self.chunks = chunks or []
+ self.error = error
+ self.calls: list[dict] = []
+
+ async def create(self, **kwargs):
+ self.calls.append(kwargs)
+ if self.error is not None:
+ raise self.error
+
+ async def generate():
+ for chunk in self.chunks:
+ if isinstance(chunk, Exception):
+ raise chunk
+ yield chunk
+
+ return generate()
+
+
+def stream_chunk(
+ content: str | None = None,
+ *,
+ finish_reason: str | None = None,
+ usage=None,
+):
+ choices = []
+ if content is not None or finish_reason is not None:
+ choices.append(
+ SimpleNamespace(
+ finish_reason=finish_reason,
+ delta=SimpleNamespace(
+ reasoning_content=None,
+ content=content,
+ tool_calls=[],
+ ),
+ )
+ )
+ return SimpleNamespace(choices=choices, usage=usage)
+
+
+def attach_client(vlm, completions: FakeCompletions) -> None:
+ vlm.get_async_client = lambda: SimpleNamespace(chat=SimpleNamespace(completions=completions))
+
+
+def make_adapter(vlm, default_model: str) -> VLMProviderAdapter:
+ return VLMProviderAdapter(
+ vlm,
+ default_model=default_model,
+ langfuse_client=SimpleNamespace(enabled=False, _client=None),
+ )
+
+
+async def test_vlm_adapter_streams_openai_content_deltas():
+ usage = SimpleNamespace(
+ prompt_tokens=3,
+ completion_tokens=2,
+ total_tokens=5,
+ prompt_tokens_details=None,
+ completion_tokens_details=None,
+ )
+ completions = FakeCompletions(
+ chunks=[
+ stream_chunk("hel"),
+ stream_chunk("lo", finish_reason="stop"),
+ stream_chunk(usage=usage),
+ ]
+ )
+ vlm = OpenAIVLM(
+ {
+ "provider": "openai",
+ "model": "gpt-test",
+ "api_key": "test-key",
+ "temperature": 0.1,
+ }
+ )
+ attach_client(vlm, completions)
+ provider = make_adapter(vlm, "gpt-test")
+
+ events = [
+ event
+ async for event in provider.chat_stream(
+ messages=[{"role": "user", "content": "hi"}],
+ )
+ ]
+
+ assert [event.type for event in events] == [
+ "content_delta",
+ "content_delta",
+ "response",
+ ]
+ assert [event.content for event in events[:-1]] == ["hel", "lo"]
+ assert events[-1].response.content == "hello"
+ assert events[-1].response.usage == {
+ "prompt_tokens": 3,
+ "completion_tokens": 2,
+ "total_tokens": 5,
+ }
+ assert completions.calls[0]["stream"] is True
+ assert completions.calls[0]["stream_options"] == {"include_usage": True}
+ assert "thinking" not in completions.calls[0]
+
+
+async def test_vlm_adapter_streams_with_credential_failover_before_first_delta():
+ primary_completions = FakeCompletions(error=RuntimeError("status 429: rate limit"))
+ backup_completions = FakeCompletions(chunks=[stream_chunk("backup", finish_reason="stop")])
+ primary = VolcEngineVLM(
+ {
+ "provider": "volcengine",
+ "model": "primary-model",
+ "api_key": "primary-key",
+ }
+ )
+ backup = VolcEngineVLM(
+ {
+ "provider": "volcengine",
+ "model": "backup-model",
+ "api_key": "backup-key",
+ }
+ )
+ attach_client(primary, primary_completions)
+ attach_client(backup, backup_completions)
+ credentials = MultiCredentialVLM(
+ [primary, backup],
+ credential_ids=["primary", "backup"],
+ )
+ provider = make_adapter(credentials, "primary-model")
+
+ events = [
+ event
+ async for event in provider.chat_stream(
+ messages=[{"role": "user", "content": "hi"}],
+ model="primary-model",
+ )
+ ]
+
+ assert [event.type for event in events] == ["content_delta", "response"]
+ assert events[0].content == "backup"
+ assert events[-1].response.content == "backup"
+ assert len(primary_completions.calls) == 1
+ assert backup_completions.calls[0]["model"] == "backup-model"
+ assert credentials.active_credential_id == "backup"
+
+
+async def test_vlm_adapter_streams_with_legacy_primary_backup_failover():
+ primary_completions = FakeCompletions(error=RuntimeError("status 401: unauthorized"))
+ backup_completions = FakeCompletions(chunks=[stream_chunk("backup", finish_reason="stop")])
+ primary = OpenAIVLM({"provider": "openai", "model": "primary-model", "api_key": "primary-key"})
+ backup = OpenAIVLM({"provider": "openai", "model": "backup-model", "api_key": "backup-key"})
+ attach_client(primary, primary_completions)
+ attach_client(backup, backup_completions)
+ failover = FailoverVLM(primary, backup)
+ provider = make_adapter(failover, "primary-model")
+
+ events = [
+ event
+ async for event in provider.chat_stream(
+ messages=[{"role": "user", "content": "hi"}],
+ )
+ ]
+
+ assert [event.type for event in events] == ["content_delta", "response"]
+ assert events[-1].response.content == "backup"
+ assert failover.is_using_backup is True
+
+
+async def test_credential_stream_does_not_replay_after_a_visible_delta():
+ primary_completions = FakeCompletions(
+ chunks=[stream_chunk("partial"), RuntimeError("status 429: rate limit")]
+ )
+ backup_completions = FakeCompletions(chunks=[stream_chunk("duplicate", finish_reason="stop")])
+ primary = OpenAIVLM({"provider": "openai", "model": "primary-model", "api_key": "primary-key"})
+ backup = OpenAIVLM({"provider": "openai", "model": "backup-model", "api_key": "backup-key"})
+ attach_client(primary, primary_completions)
+ attach_client(backup, backup_completions)
+ credentials = MultiCredentialVLM(
+ [primary, backup],
+ credential_ids=["primary", "backup"],
+ )
+ provider = make_adapter(credentials, "primary-model")
+
+ events = [
+ event
+ async for event in provider.chat_stream(
+ messages=[{"role": "user", "content": "hi"}],
+ )
+ ]
+
+ assert events[0].type == "content_delta"
+ assert events[0].content == "partial"
+ assert events[-1].type == "response"
+ assert events[-1].response.finish_reason == "error"
+ assert backup_completions.calls == []
diff --git a/openviking/models/vlm/base.py b/openviking/models/vlm/base.py
index 640e07e5c..742ac34ad 100644
--- a/openviking/models/vlm/base.py
+++ b/openviking/models/vlm/base.py
@@ -7,7 +7,7 @@ import re
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from pathlib import Path
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, AsyncIterator, Callable, Dict, List, Optional, Union
from openviking.utils.exceptions import AllCredentialsFailedError
from openviking.utils.model_retry import (
@@ -479,6 +479,45 @@ class FailoverVLM(VLMBase):
self._logger.error(f"Backup VLM also failed with error: {e}")
raise last_error
+ async def stream_with_failover(
+ self,
+ stream_factory: Callable[[VLMBase], AsyncIterator[Any]],
+ ) -> AsyncIterator[Any]:
+ """Stream from the active backend and fail over before output starts.
+
+ Once an item has been yielded, replaying the request through another
+ backend would duplicate visible output. Errors after that point are
+ therefore propagated to the caller instead of triggering failover.
+ """
+ last_error = None
+
+ if self._switcher.should_try_primary():
+ emitted = False
+ try:
+ async for item in stream_factory(self.primary):
+ emitted = True
+ yield item
+ self._switcher.record_primary_success()
+ return
+ except Exception as exc:
+ _annotate_vlm_error(exc, self.primary)
+ last_error = exc
+ if emitted or not self._switcher.record_primary_failure(exc):
+ raise
+
+ emitted = False
+ try:
+ self._switcher.record_backup_request()
+ async for item in stream_factory(self.backup):
+ emitted = True
+ yield item
+ return
+ except Exception as exc:
+ _annotate_vlm_error(exc, self.backup)
+ last_error = exc
+ self._logger.error(f"Backup VLM also failed with error: {exc}")
+ raise last_error
+
def get_completion(
self,
prompt: str = "",
@@ -743,6 +782,48 @@ class MultiCredentialVLM(VLMBase):
raise AllCredentialsFailedError(aggregated_errors)
+ async def stream_with_failover(
+ self,
+ stream_factory: Callable[[VLMBase], AsyncIterator[Any]],
+ ) -> AsyncIterator[Any]:
+ """Stream with ordered credential failover before output starts.
+
+ A credential may be replaced while opening or awaiting the first
+ visible stream item. After an item is emitted, switching would replay
+ already delivered output, so subsequent errors fail the request.
+ """
+ aggregated_errors = []
+ start = self._switcher.maybe_failback()
+ n = self._switcher.n
+
+ for offset in range(n):
+ idx = (start + offset) % n
+ credential_id = self._credential_ids[idx]
+ vlm_instance = self._vlm_instances[idx]
+ emitted = False
+
+ try:
+ async for item in stream_factory(vlm_instance):
+ emitted = True
+ yield item
+ self._switcher.commit_success(idx)
+ return
+ except Exception as exc:
+ _annotate_vlm_error(exc, vlm_instance)
+ if emitted:
+ raise
+
+ error_class = classify_api_error(exc)
+ aggregated_errors.append((credential_id, error_class, exc, idx))
+ if self._switcher.is_fail_fast(error_class):
+ raise
+
+ self._logger.warning(
+ f"Credential {credential_id} failed with {error_class}, trying next credential"
+ )
+
+ raise AllCredentialsFailedError(aggregated_errors)
+
def get_completion(
self,
prompt: str = "",