Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(vlm): add streaming response handling for OpenAI VLM (#740)

KorenKrita247293b53164
diff --git a/openviking/models/vlm/backends/openai_vlm.py b/openviking/models/vlm/backends/openai_vlm.py
index d1f6d05d3..d023a1d20 100644
--- a/openviking/models/vlm/backends/openai_vlm.py
+++ b/openviking/models/vlm/backends/openai_vlm.py
@@ -48,17 +48,179 @@ class OpenAIVLM(VLMBase):
self._async_client = openai.AsyncOpenAI(**client_kwargs)
return self._async_client
- def _update_token_usage_from_response(self, response):
- if hasattr(response, "usage") and response.usage:
- prompt_tokens = response.usage.prompt_tokens
- completion_tokens = response.usage.completion_tokens
+ def _is_streaming_response(self, response):
+ """Check if response is a streaming response.
+
+ Streaming responses are iterators that yield chunks, while non-streaming
+ responses have a choices attribute directly.
+ """
+ # Check for async streaming first to avoid false positives
+ if hasattr(response, "__aiter__"):
+ return False # Async responses handled separately
+ # Streaming responses: iterators but not strings/lists/dicts with choices
+ if hasattr(response, "__iter__") and not hasattr(response, "choices"):
+ # Exclude basic iterable types that might slip through
+ if isinstance(response, (str, bytes, list, dict)):
+ return False
+ return True
+ # Some streaming responses might have _iterator attribute
+ if hasattr(response, "_iterator") and not hasattr(response, "choices"):
+ return True
+ return False
+
+ def _is_async_streaming_response(self, response):
+ """Check if response is an async streaming response."""
+ if hasattr(response, "__aiter__") and not hasattr(response, "choices"):
+ # Exclude basic types that should never be treated as streaming
+ if isinstance(response, (str, bytes, list, dict)):
+ return False
+ return True
+ if hasattr(response, "_iterator") and not hasattr(response, "choices"):
+ return True
+ return False
+
+ def _extract_content_from_chunk(self, chunk):
+ """Extract content string from a single chunk."""
+ try:
+ choices = getattr(chunk, "choices", None)
+ if not choices:
+ return None
+ delta = getattr(choices[0], "delta", None)
+ if not delta:
+ return None
+ return getattr(delta, "content", None)
+ except (AttributeError, IndexError):
+ return None
+
+ def _extract_usage_from_chunk(self, chunk):
+ """Extract token usage from a chunk. Returns (prompt_tokens, completion_tokens)."""
+ usage = getattr(chunk, "usage", None)
+ if not usage:
+ return 0, 0
+ prompt_tokens = getattr(usage, "prompt_tokens", 0) or 0
+ completion_tokens = getattr(usage, "completion_tokens", 0) or 0
+ return prompt_tokens, completion_tokens
+
+ def _process_streaming_chunks(self, chunks):
+ """Process streaming chunks and extract content and token usage.
+
+ WARNING: This method consumes the iterator. Do not use the response
+ object after calling this method as it will be exhausted.
+
+ Returns (content, prompt_tokens, completion_tokens).
+ """
+ content_parts = []
+ prompt_tokens = 0
+ completion_tokens = 0
+
+ for chunk in chunks:
+ content = self._extract_content_from_chunk(chunk)
+ if content:
+ content_parts.append(content)
+
+ pt, ct = self._extract_usage_from_chunk(chunk)
+ if pt > 0:
+ prompt_tokens = pt
+ if ct > 0:
+ completion_tokens = ct
+
+ return "".join(content_parts), prompt_tokens, completion_tokens
+
+ def _extract_content_and_usage(self, response):
+ """Extract content from response, handling both streaming and non-streaming.
+
+ Returns (content, prompt_tokens, completion_tokens, is_streaming).
+ """
+ logger.debug(f"[OpenAIVLM] Response type: {type(response)}")
+
+ if self._is_streaming_response(response):
+ content, prompt_tokens, completion_tokens = self._process_streaming_chunks(response)
+ return content, prompt_tokens, completion_tokens, True
+ else:
+ # Non-streaming response
+ content = response.choices[0].message.content or ""
+ usage = getattr(response, "usage", None)
+ prompt_tokens = getattr(usage, "prompt_tokens", 0) or 0
+ completion_tokens = getattr(usage, "completion_tokens", 0) or 0
+ return content, prompt_tokens, completion_tokens, False
+
+ async def _extract_content_and_usage_async(self, response):
+ """Extract content from async response, handling both streaming and non-streaming.
+
+ Returns (content, prompt_tokens, completion_tokens, is_streaming).
+ """
+ logger.debug(f"[OpenAIVLM] Async response type: {type(response)}")
+
+ if self._is_async_streaming_response(response):
+ # Note: This logic mirrors _process_streaming_chunks but uses
+ # async for to handle async iterators. Python's async for and
+ # sync for cannot be unified in a single method.
+ content_parts = []
+ prompt_tokens = 0
+ completion_tokens = 0
+
+ async for chunk in response:
+ content = self._extract_content_from_chunk(chunk)
+ if content:
+ content_parts.append(content)
+
+ pt, ct = self._extract_usage_from_chunk(chunk)
+ if pt > 0:
+ prompt_tokens = pt
+ if ct > 0:
+ completion_tokens = ct
+
+ return "".join(content_parts), prompt_tokens, completion_tokens, True
+ else:
+ # Non-streaming response
+ content = response.choices[0].message.content or ""
+ usage = getattr(response, "usage", None)
+ prompt_tokens = getattr(usage, "prompt_tokens", 0) or 0
+ completion_tokens = getattr(usage, "completion_tokens", 0) or 0
+ return content, prompt_tokens, completion_tokens, False
+
+ def _finalize_response(
+ self, content, prompt_tokens, completion_tokens, is_streaming, operation_name="completion"
+ ):
+ """Finalize response: log warnings and update token usage.
+
+ Common post-processing for both sync and async responses.
+ """
+ if not content:
+ logger.warning(
+ f"[OpenAIVLM] Empty {operation_name} response received (streaming={is_streaming})"
+ )
+
+ if prompt_tokens > 0 or completion_tokens > 0:
self.update_token_usage(
model_name=self.model or "gpt-4o-mini",
provider=self.provider,
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
)
- return
+
+ return content
+
+ def _handle_response(self, response, operation_name="completion"):
+ """Handle response extraction and token usage update."""
+ content, prompt_tokens, completion_tokens, is_streaming = self._extract_content_and_usage(
+ response
+ )
+ return self._finalize_response(
+ content, prompt_tokens, completion_tokens, is_streaming, operation_name
+ )
+
+ async def _handle_response_async(self, response, operation_name="completion"):
+ """Handle async response extraction and token usage update."""
+ (
+ content,
+ prompt_tokens,
+ completion_tokens,
+ is_streaming,
+ ) = await self._extract_content_and_usage_async(response)
+ return self._finalize_response(
+ content, prompt_tokens, completion_tokens, is_streaming, operation_name
+ )
def get_completion(self, prompt: str, thinking: bool = False) -> str:
"""Get text completion"""
@@ -72,8 +234,8 @@ class OpenAIVLM(VLMBase):
kwargs["max_tokens"] = self.max_tokens
response = client.chat.completions.create(**kwargs)
- self._update_token_usage_from_response(response)
- return self._clean_response(response.choices[0].message.content or "")
+ content = self._handle_response(response, operation_name="text completion")
+ return self._clean_response(content)
async def get_completion_async(
self, prompt: str, thinking: bool = False, max_retries: int = 0
@@ -92,8 +254,10 @@ class OpenAIVLM(VLMBase):
for attempt in range(max_retries + 1):
try:
response = await client.chat.completions.create(**kwargs)
- self._update_token_usage_from_response(response)
- return self._clean_response(response.choices[0].message.content or "")
+ content = await self._handle_response_async(
+ response, operation_name="text completion"
+ )
+ return self._clean_response(content)
except Exception as e:
last_error = e
if attempt < max_retries:
@@ -179,8 +343,8 @@ class OpenAIVLM(VLMBase):
kwargs["max_tokens"] = self.max_tokens
response = client.chat.completions.create(**kwargs)
- self._update_token_usage_from_response(response)
- return self._clean_response(response.choices[0].message.content or "")
+ content = self._handle_response(response, operation_name="vision completion")
+ return self._clean_response(content)
async def get_vision_completion_async(
self,
@@ -205,5 +369,5 @@ class OpenAIVLM(VLMBase):
kwargs["max_tokens"] = self.max_tokens
response = await client.chat.completions.create(**kwargs)
- self._update_token_usage_from_response(response)
- return self._clean_response(response.choices[0].message.content or "")
+ content = await self._handle_response_async(response, operation_name="vision completion")
+ return self._clean_response(content)
diff --git a/tests/unit/test_openai_vlm_streaming.py b/tests/unit/test_openai_vlm_streaming.py
new file mode 100644
index 000000000..3ee98b3a4
--- /dev/null
+++ b/tests/unit/test_openai_vlm_streaming.py
@@ -0,0 +1,381 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: Apache-2.0
+"""Tests for OpenAI VLM streaming response handling."""
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from openviking.models.vlm.backends.openai_vlm import OpenAIVLM
+
+
+class MockChunk:
+ """Mock streaming chunk."""
+
+ def __init__(self, content=None, usage=None, finish_reason=None, empty_choices=False):
+ self.choices = []
+ if not empty_choices:
+ delta = MagicMock()
+ delta.content = content
+ delta.role = "assistant"
+ choice = MagicMock()
+ choice.delta = delta
+ choice.index = 0
+ choice.finish_reason = finish_reason
+ self.choices.append(choice)
+ self.usage = usage
+
+
+class MockUsage:
+ def __init__(self, prompt_tokens=0, completion_tokens=0):
+ self.prompt_tokens = prompt_tokens
+ self.completion_tokens = completion_tokens
+
+
+class TestOpenAIVLMStreamingDetection:
+ """Test streaming vs non-streaming detection."""
+
+ def test_is_streaming_response_iterator(self):
+ """Test detection of streaming response (has __iter__)."""
+ vlm = OpenAIVLM({"api_key": "test"})
+
+ class IteratorResponse:
+ def __iter__(self):
+ return iter([])
+
+ response = IteratorResponse()
+ assert vlm._is_streaming_response(response) is True
+
+ def test_is_streaming_response_with_choices(self):
+ """Test non-streaming response has choices attribute."""
+ vlm = OpenAIVLM({"api_key": "test"})
+
+ class ResponseWithChoices:
+ def __init__(self):
+ self.choices = [MagicMock()]
+
+ response = ResponseWithChoices()
+ assert vlm._is_streaming_response(response) is False
+
+ def test_is_streaming_response_with_iterator_attr(self):
+ """Test detection via _iterator attribute."""
+ vlm = OpenAIVLM({"api_key": "test"})
+
+ class ResponseWithIterator:
+ _iterator = True
+
+ response = ResponseWithIterator()
+ assert vlm._is_streaming_response(response) is True
+
+ def test_is_streaming_response_excludes_basic_types(self):
+ """Test that string/list/dict are not detected as streaming."""
+ vlm = OpenAIVLM({"api_key": "test"})
+
+ # Strings have __iter__ but should not be streaming
+ assert vlm._is_streaming_response("hello") is False
+ # Lists have __iter__ but should not be streaming
+ assert vlm._is_streaming_response([1, 2, 3]) is False
+ # Dicts have __iter__ but should not be streaming
+ assert vlm._is_streaming_response({"key": "value"}) is False
+ # Bytes have __iter__ but should not be streaming
+ assert vlm._is_streaming_response(b"hello") is False
+
+ def test_is_streaming_response_excludes_async(self):
+ """Test that async iterators are not detected as sync streaming."""
+ vlm = OpenAIVLM({"api_key": "test"})
+
+ class AsyncIteratorResponse:
+ def __aiter__(self):
+ return self
+
+ async def __anext__(self):
+ raise StopAsyncIteration
+
+ response = AsyncIteratorResponse()
+ # Should NOT be detected as sync streaming (has __aiter__)
+ assert vlm._is_streaming_response(response) is False
+
+ def test_is_async_streaming_response(self):
+ """Test detection of async streaming response."""
+ vlm = OpenAIVLM({"api_key": "test"})
+
+ class AsyncIteratorResponse:
+ def __aiter__(self):
+ return self
+
+ async def __anext__(self):
+ raise StopAsyncIteration
+
+ response = AsyncIteratorResponse()
+ assert vlm._is_async_streaming_response(response) is True
+
+ class ResponseWithChoices:
+ def __init__(self):
+ self.choices = [MagicMock()]
+
+ response_with_choices = ResponseWithChoices()
+ assert vlm._is_async_streaming_response(response_with_choices) is False
+
+
+class TestOpenAIVLMChunkExtraction:
+ """Test chunk content and usage extraction."""
+
+ def test_extract_content_from_chunk(self):
+ """Test extracting content from a valid chunk."""
+ vlm = OpenAIVLM({"api_key": "test"})
+ chunk = MockChunk(content="Hello world")
+ assert vlm._extract_content_from_chunk(chunk) == "Hello world"
+
+ def test_extract_content_from_chunk_empty(self):
+ """Test extracting content from empty chunk."""
+ vlm = OpenAIVLM({"api_key": "test"})
+ chunk = MockChunk(content=None)
+ assert vlm._extract_content_from_chunk(chunk) is None
+
+ def test_extract_content_from_chunk_no_choices(self):
+ """Test extracting content from chunk without choices."""
+ vlm = OpenAIVLM({"api_key": "test"})
+ chunk = MagicMock()
+ chunk.choices = []
+ assert vlm._extract_content_from_chunk(chunk) is None
+
+ def test_extract_usage_from_chunk(self):
+ """Test extracting usage from chunk."""
+ vlm = OpenAIVLM({"api_key": "test"})
+ chunk = MockChunk(usage=MockUsage(10, 20))
+ pt, ct = vlm._extract_usage_from_chunk(chunk)
+ assert pt == 10
+ assert ct == 20
+
+ def test_extract_usage_from_chunk_no_usage(self):
+ """Test extracting usage from chunk without usage."""
+ vlm = OpenAIVLM({"api_key": "test"})
+ chunk = MockChunk()
+ pt, ct = vlm._extract_usage_from_chunk(chunk)
+ assert pt == 0
+ assert ct == 0
+
+
+class TestOpenAIVLMStreamingExtraction:
+ """Test streaming response extraction."""
+
+ def test_process_streaming_chunks(self):
+ """Test processing multiple streaming chunks."""
+ vlm = OpenAIVLM({"api_key": "test"})
+
+ chunks = [
+ MockChunk(content="Hello"),
+ MockChunk(content=" "),
+ MockChunk(content="world"),
+ ]
+
+ content, pt, ct = vlm._process_streaming_chunks(chunks)
+ assert content == "Hello world"
+ assert pt == 0
+ assert ct == 0
+
+ def test_process_streaming_chunks_with_usage(self):
+ """Test processing chunks with token usage."""
+ vlm = OpenAIVLM({"api_key": "test"})
+
+ chunks = [
+ MockChunk(content="Hello", usage=MockUsage(10, 1)),
+ MockChunk(content=" world", usage=MockUsage(10, 2)),
+ MockChunk(content="!"),
+ ]
+
+ content, pt, ct = vlm._process_streaming_chunks(chunks)
+ assert content == "Hello world!"
+ # Last chunk with usage wins
+ assert pt == 10
+ assert ct == 2
+
+ def test_extract_content_and_usage_streaming(self):
+ """Test full extraction from streaming response."""
+ vlm = OpenAIVLM({"api_key": "test"})
+
+ chunks = [
+ MockChunk(content="Streamed"),
+ MockChunk(content=" content"),
+ ]
+
+ class MockStreamingResponse:
+ def __iter__(self):
+ return iter(chunks)
+
+ response = MockStreamingResponse()
+ content, pt, ct, is_streaming = vlm._extract_content_and_usage(response)
+
+ assert content == "Streamed content"
+ assert is_streaming is True
+
+ def test_extract_content_and_usage_non_streaming(self):
+ """Test full extraction from non-streaming response."""
+ vlm = OpenAIVLM({"api_key": "test"})
+
+ class NonStreamingResponse:
+ def __init__(self):
+ self.choices = [MagicMock()]
+ self.choices[0].message.content = "Normal response"
+ self.usage = MagicMock()
+ self.usage.prompt_tokens = 10
+ self.usage.completion_tokens = 5
+
+ response = NonStreamingResponse()
+ content, pt, ct, is_streaming = vlm._extract_content_and_usage(response)
+
+ assert content == "Normal response"
+ assert pt == 10
+ assert ct == 5
+ assert is_streaming is False
+
+
+class TestOpenAIVLMStreamingAsync:
+ """Test async streaming response parsing."""
+
+ @pytest.mark.asyncio
+ async def test_extract_content_and_usage_async_streaming(self):
+ """Test extracting content from async streaming response."""
+ vlm = OpenAIVLM({"api_key": "test"})
+
+ chunks = [
+ MockChunk(content="Hello"),
+ MockChunk(content=" async"),
+ MockChunk(content=" world"),
+ ]
+
+ class MockAsyncStreamingResponse:
+ def __init__(self, chunks):
+ self.chunks = chunks
+ self.index = 0
+
+ def __aiter__(self):
+ return self
+
+ async def __anext__(self):
+ if self.index >= len(self.chunks):
+ raise StopAsyncIteration
+ chunk = self.chunks[self.index]
+ self.index += 1
+ return chunk
+
+ response = MockAsyncStreamingResponse(chunks)
+ content, pt, ct, is_streaming = await vlm._extract_content_and_usage_async(response)
+
+ assert content == "Hello async world"
+ assert is_streaming is True
+
+ @pytest.mark.asyncio
+ async def test_extract_content_and_usage_async_non_streaming(self):
+ """Test extracting content from async non-streaming response."""
+ vlm = OpenAIVLM({"api_key": "test"})
+
+ class AsyncNonStreamingResponse:
+ def __init__(self):
+ self.choices = [MagicMock()]
+ self.choices[0].message.content = "Async response"
+ self.usage = None
+
+ response = AsyncNonStreamingResponse()
+ content, pt, ct, is_streaming = await vlm._extract_content_and_usage_async(response)
+
+ assert content == "Async response"
+ assert is_streaming is False
+
+
+class TestOpenAIVLMIntegration:
+ """Integration tests with mocked OpenAI client."""
+
+ @patch("openviking.models.vlm.backends.openai_vlm.openai.OpenAI")
+ def test_get_completion_with_streaming_response(self, mock_openai_class):
+ """Test get_completion when API returns streaming format."""
+ mock_client = MagicMock()
+ mock_openai_class.return_value = mock_client
+
+ chunks = [
+ MockChunk(content="Streamed"),
+ MockChunk(content=" response"),
+ MockChunk(content=" text"),
+ ]
+
+ class MockStream:
+ def __iter__(self):
+ return iter(chunks)
+
+ mock_client.chat.completions.create.return_value = MockStream()
+
+ vlm = OpenAIVLM({"api_key": "test", "model": "gpt-4o-mini"})
+ result = vlm.get_completion("Hello")
+
+ assert result == "Streamed response text"
+
+ @patch("openviking.models.vlm.backends.openai_vlm.openai.OpenAI")
+ def test_get_completion_with_non_streaming_response(self, mock_openai_class):
+ """Test get_completion when API returns normal format."""
+ mock_client = MagicMock()
+ mock_openai_class.return_value = mock_client
+
+ class NonStreamingResponse:
+ def __init__(self):
+ self.choices = [MagicMock()]
+ self.choices[0].message.content = "Normal response"
+ self.usage = MagicMock(prompt_tokens=10, completion_tokens=5)
+
+ mock_client.chat.completions.create.return_value = NonStreamingResponse()
+
+ vlm = OpenAIVLM({"api_key": "test", "model": "gpt-4o-mini"})
+ result = vlm.get_completion("Hello")
+
+ assert result == "Normal response"
+ # Verify token usage was updated
+ usage = vlm.get_token_usage_summary()
+ assert usage["total_prompt_tokens"] == 10
+ assert usage["total_completion_tokens"] == 5
+
+ @patch("openviking.models.vlm.backends.openai_vlm.openai.OpenAI")
+ def test_get_vision_completion_with_streaming(self, mock_openai_class):
+ """Test vision completion with streaming response."""
+ mock_client = MagicMock()
+ mock_openai_class.return_value = mock_client
+
+ chunks = [
+ MockChunk(content="Vision"),
+ MockChunk(content=" result"),
+ ]
+
+ class MockStream:
+ def __iter__(self):
+ return iter(chunks)
+
+ mock_client.chat.completions.create.return_value = MockStream()
+
+ vlm = OpenAIVLM({"api_key": "test", "model": "gpt-4o-mini"})
+ result = vlm.get_vision_completion("Describe this", ["image_url"])
+
+ assert result == "Vision result"
+
+ @patch("openviking.models.vlm.backends.openai_vlm.openai.OpenAI")
+ def test_token_usage_with_streaming_chunks(self, mock_openai_class):
+ """Test token usage extraction from streaming chunks."""
+ mock_client = MagicMock()
+ mock_openai_class.return_value = mock_client
+
+ chunks = [
+ MockChunk(content="Hello", usage=MockUsage(40, 1)),
+ MockChunk(content=" world", usage=MockUsage(40, 2)),
+ MockChunk(content="!", usage=MockUsage(40, 3)),
+ ]
+
+ class MockStream:
+ def __iter__(self):
+ return iter(chunks)
+
+ mock_client.chat.completions.create.return_value = MockStream()
+
+ vlm = OpenAIVLM({"api_key": "test", "model": "test-model"})
+ result = vlm.get_completion("Hello")
+
+ assert result == "Hello world!"
+ usage = vlm.get_token_usage_summary()
+ assert usage["total_prompt_tokens"] == 40
+ assert usage["total_completion_tokens"] == 3