Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

Revert "feat(vlm): add streaming response handling for OpenAI VLM (#740)" (#745)

KorenKrita2294355f2f8f
diff --git a/openviking/models/vlm/backends/openai_vlm.py b/openviking/models/vlm/backends/openai_vlm.py
index d023a1d20..d1f6d05d3 100644
--- a/openviking/models/vlm/backends/openai_vlm.py
+++ b/openviking/models/vlm/backends/openai_vlm.py
@@ -48,179 +48,17 @@ class OpenAIVLM(VLMBase):
self._async_client = openai.AsyncOpenAI(**client_kwargs)
return self._async_client
- def _is_streaming_response(self, response):
- """Check if response is a streaming response.
-
- Streaming responses are iterators that yield chunks, while non-streaming
- responses have a choices attribute directly.
- """
- # Check for async streaming first to avoid false positives
- if hasattr(response, "__aiter__"):
- return False # Async responses handled separately
- # Streaming responses: iterators but not strings/lists/dicts with choices
- if hasattr(response, "__iter__") and not hasattr(response, "choices"):
- # Exclude basic iterable types that might slip through
- if isinstance(response, (str, bytes, list, dict)):
- return False
- return True
- # Some streaming responses might have _iterator attribute
- if hasattr(response, "_iterator") and not hasattr(response, "choices"):
- return True
- return False
-
- def _is_async_streaming_response(self, response):
- """Check if response is an async streaming response."""
- if hasattr(response, "__aiter__") and not hasattr(response, "choices"):
- # Exclude basic types that should never be treated as streaming
- if isinstance(response, (str, bytes, list, dict)):
- return False
- return True
- if hasattr(response, "_iterator") and not hasattr(response, "choices"):
- return True
- return False
-
- def _extract_content_from_chunk(self, chunk):
- """Extract content string from a single chunk."""
- try:
- choices = getattr(chunk, "choices", None)
- if not choices:
- return None
- delta = getattr(choices[0], "delta", None)
- if not delta:
- return None
- return getattr(delta, "content", None)
- except (AttributeError, IndexError):
- return None
-
- def _extract_usage_from_chunk(self, chunk):
- """Extract token usage from a chunk. Returns (prompt_tokens, completion_tokens)."""
- usage = getattr(chunk, "usage", None)
- if not usage:
- return 0, 0
- prompt_tokens = getattr(usage, "prompt_tokens", 0) or 0
- completion_tokens = getattr(usage, "completion_tokens", 0) or 0
- return prompt_tokens, completion_tokens
-
- def _process_streaming_chunks(self, chunks):
- """Process streaming chunks and extract content and token usage.
-
- WARNING: This method consumes the iterator. Do not use the response
- object after calling this method as it will be exhausted.
-
- Returns (content, prompt_tokens, completion_tokens).
- """
- content_parts = []
- prompt_tokens = 0
- completion_tokens = 0
-
- for chunk in chunks:
- content = self._extract_content_from_chunk(chunk)
- if content:
- content_parts.append(content)
-
- pt, ct = self._extract_usage_from_chunk(chunk)
- if pt > 0:
- prompt_tokens = pt
- if ct > 0:
- completion_tokens = ct
-
- return "".join(content_parts), prompt_tokens, completion_tokens
-
- def _extract_content_and_usage(self, response):
- """Extract content from response, handling both streaming and non-streaming.
-
- Returns (content, prompt_tokens, completion_tokens, is_streaming).
- """
- logger.debug(f"[OpenAIVLM] Response type: {type(response)}")
-
- if self._is_streaming_response(response):
- content, prompt_tokens, completion_tokens = self._process_streaming_chunks(response)
- return content, prompt_tokens, completion_tokens, True
- else:
- # Non-streaming response
- content = response.choices[0].message.content or ""
- usage = getattr(response, "usage", None)
- prompt_tokens = getattr(usage, "prompt_tokens", 0) or 0
- completion_tokens = getattr(usage, "completion_tokens", 0) or 0
- return content, prompt_tokens, completion_tokens, False
-
- async def _extract_content_and_usage_async(self, response):
- """Extract content from async response, handling both streaming and non-streaming.
-
- Returns (content, prompt_tokens, completion_tokens, is_streaming).
- """
- logger.debug(f"[OpenAIVLM] Async response type: {type(response)}")
-
- if self._is_async_streaming_response(response):
- # Note: This logic mirrors _process_streaming_chunks but uses
- # async for to handle async iterators. Python's async for and
- # sync for cannot be unified in a single method.
- content_parts = []
- prompt_tokens = 0
- completion_tokens = 0
-
- async for chunk in response:
- content = self._extract_content_from_chunk(chunk)
- if content:
- content_parts.append(content)
-
- pt, ct = self._extract_usage_from_chunk(chunk)
- if pt > 0:
- prompt_tokens = pt
- if ct > 0:
- completion_tokens = ct
-
- return "".join(content_parts), prompt_tokens, completion_tokens, True
- else:
- # Non-streaming response
- content = response.choices[0].message.content or ""
- usage = getattr(response, "usage", None)
- prompt_tokens = getattr(usage, "prompt_tokens", 0) or 0
- completion_tokens = getattr(usage, "completion_tokens", 0) or 0
- return content, prompt_tokens, completion_tokens, False
-
- def _finalize_response(
- self, content, prompt_tokens, completion_tokens, is_streaming, operation_name="completion"
- ):
- """Finalize response: log warnings and update token usage.
-
- Common post-processing for both sync and async responses.
- """
- if not content:
- logger.warning(
- f"[OpenAIVLM] Empty {operation_name} response received (streaming={is_streaming})"
- )
-
- if prompt_tokens > 0 or completion_tokens > 0:
+ def _update_token_usage_from_response(self, response):
+ if hasattr(response, "usage") and response.usage:
+ prompt_tokens = response.usage.prompt_tokens
+ completion_tokens = response.usage.completion_tokens
self.update_token_usage(
model_name=self.model or "gpt-4o-mini",
provider=self.provider,
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
)
-
- return content
-
- def _handle_response(self, response, operation_name="completion"):
- """Handle response extraction and token usage update."""
- content, prompt_tokens, completion_tokens, is_streaming = self._extract_content_and_usage(
- response
- )
- return self._finalize_response(
- content, prompt_tokens, completion_tokens, is_streaming, operation_name
- )
-
- async def _handle_response_async(self, response, operation_name="completion"):
- """Handle async response extraction and token usage update."""
- (
- content,
- prompt_tokens,
- completion_tokens,
- is_streaming,
- ) = await self._extract_content_and_usage_async(response)
- return self._finalize_response(
- content, prompt_tokens, completion_tokens, is_streaming, operation_name
- )
+ return
def get_completion(self, prompt: str, thinking: bool = False) -> str:
"""Get text completion"""
@@ -234,8 +72,8 @@ class OpenAIVLM(VLMBase):
kwargs["max_tokens"] = self.max_tokens
response = client.chat.completions.create(**kwargs)
- content = self._handle_response(response, operation_name="text completion")
- return self._clean_response(content)
+ self._update_token_usage_from_response(response)
+ return self._clean_response(response.choices[0].message.content or "")
async def get_completion_async(
self, prompt: str, thinking: bool = False, max_retries: int = 0
@@ -254,10 +92,8 @@ class OpenAIVLM(VLMBase):
for attempt in range(max_retries + 1):
try:
response = await client.chat.completions.create(**kwargs)
- content = await self._handle_response_async(
- response, operation_name="text completion"
- )
- return self._clean_response(content)
+ self._update_token_usage_from_response(response)
+ return self._clean_response(response.choices[0].message.content or "")
except Exception as e:
last_error = e
if attempt < max_retries:
@@ -343,8 +179,8 @@ class OpenAIVLM(VLMBase):
kwargs["max_tokens"] = self.max_tokens
response = client.chat.completions.create(**kwargs)
- content = self._handle_response(response, operation_name="vision completion")
- return self._clean_response(content)
+ self._update_token_usage_from_response(response)
+ return self._clean_response(response.choices[0].message.content or "")
async def get_vision_completion_async(
self,
@@ -369,5 +205,5 @@ class OpenAIVLM(VLMBase):
kwargs["max_tokens"] = self.max_tokens
response = await client.chat.completions.create(**kwargs)
- content = await self._handle_response_async(response, operation_name="vision completion")
- return self._clean_response(content)
+ self._update_token_usage_from_response(response)
+ return self._clean_response(response.choices[0].message.content or "")
diff --git a/tests/unit/test_openai_vlm_streaming.py b/tests/unit/test_openai_vlm_streaming.py
deleted file mode 100644
index 3ee98b3a4..000000000
--- a/tests/unit/test_openai_vlm_streaming.py
+++ /dev/null
@@ -1,381 +0,0 @@
-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
-# SPDX-License-Identifier: Apache-2.0
-"""Tests for OpenAI VLM streaming response handling."""
-
-from unittest.mock import MagicMock, patch
-
-import pytest
-
-from openviking.models.vlm.backends.openai_vlm import OpenAIVLM
-
-
-class MockChunk:
- """Mock streaming chunk."""
-
- def __init__(self, content=None, usage=None, finish_reason=None, empty_choices=False):
- self.choices = []
- if not empty_choices:
- delta = MagicMock()
- delta.content = content
- delta.role = "assistant"
- choice = MagicMock()
- choice.delta = delta
- choice.index = 0
- choice.finish_reason = finish_reason
- self.choices.append(choice)
- self.usage = usage
-
-
-class MockUsage:
- def __init__(self, prompt_tokens=0, completion_tokens=0):
- self.prompt_tokens = prompt_tokens
- self.completion_tokens = completion_tokens
-
-
-class TestOpenAIVLMStreamingDetection:
- """Test streaming vs non-streaming detection."""
-
- def test_is_streaming_response_iterator(self):
- """Test detection of streaming response (has __iter__)."""
- vlm = OpenAIVLM({"api_key": "test"})
-
- class IteratorResponse:
- def __iter__(self):
- return iter([])
-
- response = IteratorResponse()
- assert vlm._is_streaming_response(response) is True
-
- def test_is_streaming_response_with_choices(self):
- """Test non-streaming response has choices attribute."""
- vlm = OpenAIVLM({"api_key": "test"})
-
- class ResponseWithChoices:
- def __init__(self):
- self.choices = [MagicMock()]
-
- response = ResponseWithChoices()
- assert vlm._is_streaming_response(response) is False
-
- def test_is_streaming_response_with_iterator_attr(self):
- """Test detection via _iterator attribute."""
- vlm = OpenAIVLM({"api_key": "test"})
-
- class ResponseWithIterator:
- _iterator = True
-
- response = ResponseWithIterator()
- assert vlm._is_streaming_response(response) is True
-
- def test_is_streaming_response_excludes_basic_types(self):
- """Test that string/list/dict are not detected as streaming."""
- vlm = OpenAIVLM({"api_key": "test"})
-
- # Strings have __iter__ but should not be streaming
- assert vlm._is_streaming_response("hello") is False
- # Lists have __iter__ but should not be streaming
- assert vlm._is_streaming_response([1, 2, 3]) is False
- # Dicts have __iter__ but should not be streaming
- assert vlm._is_streaming_response({"key": "value"}) is False
- # Bytes have __iter__ but should not be streaming
- assert vlm._is_streaming_response(b"hello") is False
-
- def test_is_streaming_response_excludes_async(self):
- """Test that async iterators are not detected as sync streaming."""
- vlm = OpenAIVLM({"api_key": "test"})
-
- class AsyncIteratorResponse:
- def __aiter__(self):
- return self
-
- async def __anext__(self):
- raise StopAsyncIteration
-
- response = AsyncIteratorResponse()
- # Should NOT be detected as sync streaming (has __aiter__)
- assert vlm._is_streaming_response(response) is False
-
- def test_is_async_streaming_response(self):
- """Test detection of async streaming response."""
- vlm = OpenAIVLM({"api_key": "test"})
-
- class AsyncIteratorResponse:
- def __aiter__(self):
- return self
-
- async def __anext__(self):
- raise StopAsyncIteration
-
- response = AsyncIteratorResponse()
- assert vlm._is_async_streaming_response(response) is True
-
- class ResponseWithChoices:
- def __init__(self):
- self.choices = [MagicMock()]
-
- response_with_choices = ResponseWithChoices()
- assert vlm._is_async_streaming_response(response_with_choices) is False
-
-
-class TestOpenAIVLMChunkExtraction:
- """Test chunk content and usage extraction."""
-
- def test_extract_content_from_chunk(self):
- """Test extracting content from a valid chunk."""
- vlm = OpenAIVLM({"api_key": "test"})
- chunk = MockChunk(content="Hello world")
- assert vlm._extract_content_from_chunk(chunk) == "Hello world"
-
- def test_extract_content_from_chunk_empty(self):
- """Test extracting content from empty chunk."""
- vlm = OpenAIVLM({"api_key": "test"})
- chunk = MockChunk(content=None)
- assert vlm._extract_content_from_chunk(chunk) is None
-
- def test_extract_content_from_chunk_no_choices(self):
- """Test extracting content from chunk without choices."""
- vlm = OpenAIVLM({"api_key": "test"})
- chunk = MagicMock()
- chunk.choices = []
- assert vlm._extract_content_from_chunk(chunk) is None
-
- def test_extract_usage_from_chunk(self):
- """Test extracting usage from chunk."""
- vlm = OpenAIVLM({"api_key": "test"})
- chunk = MockChunk(usage=MockUsage(10, 20))
- pt, ct = vlm._extract_usage_from_chunk(chunk)
- assert pt == 10
- assert ct == 20
-
- def test_extract_usage_from_chunk_no_usage(self):
- """Test extracting usage from chunk without usage."""
- vlm = OpenAIVLM({"api_key": "test"})
- chunk = MockChunk()
- pt, ct = vlm._extract_usage_from_chunk(chunk)
- assert pt == 0
- assert ct == 0
-
-
-class TestOpenAIVLMStreamingExtraction:
- """Test streaming response extraction."""
-
- def test_process_streaming_chunks(self):
- """Test processing multiple streaming chunks."""
- vlm = OpenAIVLM({"api_key": "test"})
-
- chunks = [
- MockChunk(content="Hello"),
- MockChunk(content=" "),
- MockChunk(content="world"),
- ]
-
- content, pt, ct = vlm._process_streaming_chunks(chunks)
- assert content == "Hello world"
- assert pt == 0
- assert ct == 0
-
- def test_process_streaming_chunks_with_usage(self):
- """Test processing chunks with token usage."""
- vlm = OpenAIVLM({"api_key": "test"})
-
- chunks = [
- MockChunk(content="Hello", usage=MockUsage(10, 1)),
- MockChunk(content=" world", usage=MockUsage(10, 2)),
- MockChunk(content="!"),
- ]
-
- content, pt, ct = vlm._process_streaming_chunks(chunks)
- assert content == "Hello world!"
- # Last chunk with usage wins
- assert pt == 10
- assert ct == 2
-
- def test_extract_content_and_usage_streaming(self):
- """Test full extraction from streaming response."""
- vlm = OpenAIVLM({"api_key": "test"})
-
- chunks = [
- MockChunk(content="Streamed"),
- MockChunk(content=" content"),
- ]
-
- class MockStreamingResponse:
- def __iter__(self):
- return iter(chunks)
-
- response = MockStreamingResponse()
- content, pt, ct, is_streaming = vlm._extract_content_and_usage(response)
-
- assert content == "Streamed content"
- assert is_streaming is True
-
- def test_extract_content_and_usage_non_streaming(self):
- """Test full extraction from non-streaming response."""
- vlm = OpenAIVLM({"api_key": "test"})
-
- class NonStreamingResponse:
- def __init__(self):
- self.choices = [MagicMock()]
- self.choices[0].message.content = "Normal response"
- self.usage = MagicMock()
- self.usage.prompt_tokens = 10
- self.usage.completion_tokens = 5
-
- response = NonStreamingResponse()
- content, pt, ct, is_streaming = vlm._extract_content_and_usage(response)
-
- assert content == "Normal response"
- assert pt == 10
- assert ct == 5
- assert is_streaming is False
-
-
-class TestOpenAIVLMStreamingAsync:
- """Test async streaming response parsing."""
-
- @pytest.mark.asyncio
- async def test_extract_content_and_usage_async_streaming(self):
- """Test extracting content from async streaming response."""
- vlm = OpenAIVLM({"api_key": "test"})
-
- chunks = [
- MockChunk(content="Hello"),
- MockChunk(content=" async"),
- MockChunk(content=" world"),
- ]
-
- class MockAsyncStreamingResponse:
- def __init__(self, chunks):
- self.chunks = chunks
- self.index = 0
-
- def __aiter__(self):
- return self
-
- async def __anext__(self):
- if self.index >= len(self.chunks):
- raise StopAsyncIteration
- chunk = self.chunks[self.index]
- self.index += 1
- return chunk
-
- response = MockAsyncStreamingResponse(chunks)
- content, pt, ct, is_streaming = await vlm._extract_content_and_usage_async(response)
-
- assert content == "Hello async world"
- assert is_streaming is True
-
- @pytest.mark.asyncio
- async def test_extract_content_and_usage_async_non_streaming(self):
- """Test extracting content from async non-streaming response."""
- vlm = OpenAIVLM({"api_key": "test"})
-
- class AsyncNonStreamingResponse:
- def __init__(self):
- self.choices = [MagicMock()]
- self.choices[0].message.content = "Async response"
- self.usage = None
-
- response = AsyncNonStreamingResponse()
- content, pt, ct, is_streaming = await vlm._extract_content_and_usage_async(response)
-
- assert content == "Async response"
- assert is_streaming is False
-
-
-class TestOpenAIVLMIntegration:
- """Integration tests with mocked OpenAI client."""
-
- @patch("openviking.models.vlm.backends.openai_vlm.openai.OpenAI")
- def test_get_completion_with_streaming_response(self, mock_openai_class):
- """Test get_completion when API returns streaming format."""
- mock_client = MagicMock()
- mock_openai_class.return_value = mock_client
-
- chunks = [
- MockChunk(content="Streamed"),
- MockChunk(content=" response"),
- MockChunk(content=" text"),
- ]
-
- class MockStream:
- def __iter__(self):
- return iter(chunks)
-
- mock_client.chat.completions.create.return_value = MockStream()
-
- vlm = OpenAIVLM({"api_key": "test", "model": "gpt-4o-mini"})
- result = vlm.get_completion("Hello")
-
- assert result == "Streamed response text"
-
- @patch("openviking.models.vlm.backends.openai_vlm.openai.OpenAI")
- def test_get_completion_with_non_streaming_response(self, mock_openai_class):
- """Test get_completion when API returns normal format."""
- mock_client = MagicMock()
- mock_openai_class.return_value = mock_client
-
- class NonStreamingResponse:
- def __init__(self):
- self.choices = [MagicMock()]
- self.choices[0].message.content = "Normal response"
- self.usage = MagicMock(prompt_tokens=10, completion_tokens=5)
-
- mock_client.chat.completions.create.return_value = NonStreamingResponse()
-
- vlm = OpenAIVLM({"api_key": "test", "model": "gpt-4o-mini"})
- result = vlm.get_completion("Hello")
-
- assert result == "Normal response"
- # Verify token usage was updated
- usage = vlm.get_token_usage_summary()
- assert usage["total_prompt_tokens"] == 10
- assert usage["total_completion_tokens"] == 5
-
- @patch("openviking.models.vlm.backends.openai_vlm.openai.OpenAI")
- def test_get_vision_completion_with_streaming(self, mock_openai_class):
- """Test vision completion with streaming response."""
- mock_client = MagicMock()
- mock_openai_class.return_value = mock_client
-
- chunks = [
- MockChunk(content="Vision"),
- MockChunk(content=" result"),
- ]
-
- class MockStream:
- def __iter__(self):
- return iter(chunks)
-
- mock_client.chat.completions.create.return_value = MockStream()
-
- vlm = OpenAIVLM({"api_key": "test", "model": "gpt-4o-mini"})
- result = vlm.get_vision_completion("Describe this", ["image_url"])
-
- assert result == "Vision result"
-
- @patch("openviking.models.vlm.backends.openai_vlm.openai.OpenAI")
- def test_token_usage_with_streaming_chunks(self, mock_openai_class):
- """Test token usage extraction from streaming chunks."""
- mock_client = MagicMock()
- mock_openai_class.return_value = mock_client
-
- chunks = [
- MockChunk(content="Hello", usage=MockUsage(40, 1)),
- MockChunk(content=" world", usage=MockUsage(40, 2)),
- MockChunk(content="!", usage=MockUsage(40, 3)),
- ]
-
- class MockStream:
- def __iter__(self):
- return iter(chunks)
-
- mock_client.chat.completions.create.return_value = MockStream()
-
- vlm = OpenAIVLM({"api_key": "test", "model": "test-model"})
- result = vlm.get_completion("Hello")
-
- assert result == "Hello world!"
- usage = vlm.get_token_usage_summary()
- assert usage["total_prompt_tokens"] == 40
- assert usage["total_completion_tokens"] == 3