Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(vlm): add streaming response handling for OpenAI VLM (#740)
KorenKrita247293b53164
diff --git a/openviking/models/vlm/backends/openai_vlm.py b/openviking/models/vlm/backends/openai_vlm.pyindex d1f6d05d3..d023a1d20 100644--- a/openviking/models/vlm/backends/openai_vlm.py+++ b/openviking/models/vlm/backends/openai_vlm.py@@ -48,17 +48,179 @@ class OpenAIVLM(VLMBase):self._async_client = openai.AsyncOpenAI(**client_kwargs)return self._async_client- def _update_token_usage_from_response(self, response):- if hasattr(response, "usage") and response.usage:- prompt_tokens = response.usage.prompt_tokens- completion_tokens = response.usage.completion_tokens+ def _is_streaming_response(self, response):+ """Check if response is a streaming response.++ Streaming responses are iterators that yield chunks, while non-streaming+ responses have a choices attribute directly.+ """+ # Check for async streaming first to avoid false positives+ if hasattr(response, "__aiter__"):+ return False # Async responses handled separately+ # Streaming responses: iterators but not strings/lists/dicts with choices+ if hasattr(response, "__iter__") and not hasattr(response, "choices"):+ # Exclude basic iterable types that might slip through+ if isinstance(response, (str, bytes, list, dict)):+ return False+ return True+ # Some streaming responses might have _iterator attribute+ if hasattr(response, "_iterator") and not hasattr(response, "choices"):+ return True+ return False++ def _is_async_streaming_response(self, response):+ """Check if response is an async streaming response."""+ if hasattr(response, "__aiter__") and not hasattr(response, "choices"):+ # Exclude basic types that should never be treated as streaming+ if isinstance(response, (str, bytes, list, dict)):+ return False+ return True+ if hasattr(response, "_iterator") and not hasattr(response, "choices"):+ return True+ return False++ def _extract_content_from_chunk(self, chunk):+ """Extract content string from a single chunk."""+ try:+ choices = getattr(chunk, "choices", None)+ if not choices:+ return None+ delta = getattr(choices[0], "delta", None)+ if not delta:+ return None+ return getattr(delta, "content", None)+ except (AttributeError, IndexError):+ return None++ def _extract_usage_from_chunk(self, chunk):+ """Extract token usage from a chunk. Returns (prompt_tokens, completion_tokens)."""+ usage = getattr(chunk, "usage", None)+ if not usage:+ return 0, 0+ prompt_tokens = getattr(usage, "prompt_tokens", 0) or 0+ completion_tokens = getattr(usage, "completion_tokens", 0) or 0+ return prompt_tokens, completion_tokens++ def _process_streaming_chunks(self, chunks):+ """Process streaming chunks and extract content and token usage.++ WARNING: This method consumes the iterator. Do not use the response+ object after calling this method as it will be exhausted.++ Returns (content, prompt_tokens, completion_tokens).+ """+ content_parts = []+ prompt_tokens = 0+ completion_tokens = 0++ for chunk in chunks:+ content = self._extract_content_from_chunk(chunk)+ if content:+ content_parts.append(content)++ pt, ct = self._extract_usage_from_chunk(chunk)+ if pt > 0:+ prompt_tokens = pt+ if ct > 0:+ completion_tokens = ct++ return "".join(content_parts), prompt_tokens, completion_tokens++ def _extract_content_and_usage(self, response):+ """Extract content from response, handling both streaming and non-streaming.++ Returns (content, prompt_tokens, completion_tokens, is_streaming).+ """+ logger.debug(f"[OpenAIVLM] Response type: {type(response)}")++ if self._is_streaming_response(response):+ content, prompt_tokens, completion_tokens = self._process_streaming_chunks(response)+ return content, prompt_tokens, completion_tokens, True+ else:+ # Non-streaming response+ content = response.choices[0].message.content or ""+ usage = getattr(response, "usage", None)+ prompt_tokens = getattr(usage, "prompt_tokens", 0) or 0+ completion_tokens = getattr(usage, "completion_tokens", 0) or 0+ return content, prompt_tokens, completion_tokens, False++ async def _extract_content_and_usage_async(self, response):+ """Extract content from async response, handling both streaming and non-streaming.++ Returns (content, prompt_tokens, completion_tokens, is_streaming).+ """+ logger.debug(f"[OpenAIVLM] Async response type: {type(response)}")++ if self._is_async_streaming_response(response):+ # Note: This logic mirrors _process_streaming_chunks but uses+ # async for to handle async iterators. Python's async for and+ # sync for cannot be unified in a single method.+ content_parts = []+ prompt_tokens = 0+ completion_tokens = 0++ async for chunk in response:+ content = self._extract_content_from_chunk(chunk)+ if content:+ content_parts.append(content)++ pt, ct = self._extract_usage_from_chunk(chunk)+ if pt > 0:+ prompt_tokens = pt+ if ct > 0:+ completion_tokens = ct++ return "".join(content_parts), prompt_tokens, completion_tokens, True+ else:+ # Non-streaming response+ content = response.choices[0].message.content or ""+ usage = getattr(response, "usage", None)+ prompt_tokens = getattr(usage, "prompt_tokens", 0) or 0+ completion_tokens = getattr(usage, "completion_tokens", 0) or 0+ return content, prompt_tokens, completion_tokens, False++ def _finalize_response(+ self, content, prompt_tokens, completion_tokens, is_streaming, operation_name="completion"+ ):+ """Finalize response: log warnings and update token usage.++ Common post-processing for both sync and async responses.+ """+ if not content:+ logger.warning(+ f"[OpenAIVLM] Empty {operation_name} response received (streaming={is_streaming})"+ )++ if prompt_tokens > 0 or completion_tokens > 0:self.update_token_usage(model_name=self.model or "gpt-4o-mini",provider=self.provider,prompt_tokens=prompt_tokens,completion_tokens=completion_tokens,)- return++ return content++ def _handle_response(self, response, operation_name="completion"):+ """Handle response extraction and token usage update."""+ content, prompt_tokens, completion_tokens, is_streaming = self._extract_content_and_usage(+ response+ )+ return self._finalize_response(+ content, prompt_tokens, completion_tokens, is_streaming, operation_name+ )++ async def _handle_response_async(self, response, operation_name="completion"):+ """Handle async response extraction and token usage update."""+ (+ content,+ prompt_tokens,+ completion_tokens,+ is_streaming,+ ) = await self._extract_content_and_usage_async(response)+ return self._finalize_response(+ content, prompt_tokens, completion_tokens, is_streaming, operation_name+ )def get_completion(self, prompt: str, thinking: bool = False) -> str:"""Get text completion"""@@ -72,8 +234,8 @@ class OpenAIVLM(VLMBase):kwargs["max_tokens"] = self.max_tokensresponse = client.chat.completions.create(**kwargs)- self._update_token_usage_from_response(response)- return self._clean_response(response.choices[0].message.content or "")+ content = self._handle_response(response, operation_name="text completion")+ return self._clean_response(content)async def get_completion_async(self, prompt: str, thinking: bool = False, max_retries: int = 0@@ -92,8 +254,10 @@ class OpenAIVLM(VLMBase):for attempt in range(max_retries + 1):try:response = await client.chat.completions.create(**kwargs)- self._update_token_usage_from_response(response)- return self._clean_response(response.choices[0].message.content or "")+ content = await self._handle_response_async(+ response, operation_name="text completion"+ )+ return self._clean_response(content)except Exception as e:last_error = eif attempt < max_retries:@@ -179,8 +343,8 @@ class OpenAIVLM(VLMBase):kwargs["max_tokens"] = self.max_tokensresponse = client.chat.completions.create(**kwargs)- self._update_token_usage_from_response(response)- return self._clean_response(response.choices[0].message.content or "")+ content = self._handle_response(response, operation_name="vision completion")+ return self._clean_response(content)async def get_vision_completion_async(self,@@ -205,5 +369,5 @@ class OpenAIVLM(VLMBase):kwargs["max_tokens"] = self.max_tokensresponse = await client.chat.completions.create(**kwargs)- self._update_token_usage_from_response(response)- return self._clean_response(response.choices[0].message.content or "")+ content = await self._handle_response_async(response, operation_name="vision completion")+ return self._clean_response(content)diff --git a/tests/unit/test_openai_vlm_streaming.py b/tests/unit/test_openai_vlm_streaming.pynew file mode 100644index 000000000..3ee98b3a4--- /dev/null+++ b/tests/unit/test_openai_vlm_streaming.py@@ -0,0 +1,381 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: Apache-2.0+"""Tests for OpenAI VLM streaming response handling."""++from unittest.mock import MagicMock, patch++import pytest++from openviking.models.vlm.backends.openai_vlm import OpenAIVLM+++class MockChunk:+ """Mock streaming chunk."""++ def __init__(self, content=None, usage=None, finish_reason=None, empty_choices=False):+ self.choices = []+ if not empty_choices:+ delta = MagicMock()+ delta.content = content+ delta.role = "assistant"+ choice = MagicMock()+ choice.delta = delta+ choice.index = 0+ choice.finish_reason = finish_reason+ self.choices.append(choice)+ self.usage = usage+++class MockUsage:+ def __init__(self, prompt_tokens=0, completion_tokens=0):+ self.prompt_tokens = prompt_tokens+ self.completion_tokens = completion_tokens+++class TestOpenAIVLMStreamingDetection:+ """Test streaming vs non-streaming detection."""++ def test_is_streaming_response_iterator(self):+ """Test detection of streaming response (has __iter__)."""+ vlm = OpenAIVLM({"api_key": "test"})++ class IteratorResponse:+ def __iter__(self):+ return iter([])++ response = IteratorResponse()+ assert vlm._is_streaming_response(response) is True++ def test_is_streaming_response_with_choices(self):+ """Test non-streaming response has choices attribute."""+ vlm = OpenAIVLM({"api_key": "test"})++ class ResponseWithChoices:+ def __init__(self):+ self.choices = [MagicMock()]++ response = ResponseWithChoices()+ assert vlm._is_streaming_response(response) is False++ def test_is_streaming_response_with_iterator_attr(self):+ """Test detection via _iterator attribute."""+ vlm = OpenAIVLM({"api_key": "test"})++ class ResponseWithIterator:+ _iterator = True++ response = ResponseWithIterator()+ assert vlm._is_streaming_response(response) is True++ def test_is_streaming_response_excludes_basic_types(self):+ """Test that string/list/dict are not detected as streaming."""+ vlm = OpenAIVLM({"api_key": "test"})++ # Strings have __iter__ but should not be streaming+ assert vlm._is_streaming_response("hello") is False+ # Lists have __iter__ but should not be streaming+ assert vlm._is_streaming_response([1, 2, 3]) is False+ # Dicts have __iter__ but should not be streaming+ assert vlm._is_streaming_response({"key": "value"}) is False+ # Bytes have __iter__ but should not be streaming+ assert vlm._is_streaming_response(b"hello") is False++ def test_is_streaming_response_excludes_async(self):+ """Test that async iterators are not detected as sync streaming."""+ vlm = OpenAIVLM({"api_key": "test"})++ class AsyncIteratorResponse:+ def __aiter__(self):+ return self++ async def __anext__(self):+ raise StopAsyncIteration++ response = AsyncIteratorResponse()+ # Should NOT be detected as sync streaming (has __aiter__)+ assert vlm._is_streaming_response(response) is False++ def test_is_async_streaming_response(self):+ """Test detection of async streaming response."""+ vlm = OpenAIVLM({"api_key": "test"})++ class AsyncIteratorResponse:+ def __aiter__(self):+ return self++ async def __anext__(self):+ raise StopAsyncIteration++ response = AsyncIteratorResponse()+ assert vlm._is_async_streaming_response(response) is True++ class ResponseWithChoices:+ def __init__(self):+ self.choices = [MagicMock()]++ response_with_choices = ResponseWithChoices()+ assert vlm._is_async_streaming_response(response_with_choices) is False+++class TestOpenAIVLMChunkExtraction:+ """Test chunk content and usage extraction."""++ def test_extract_content_from_chunk(self):+ """Test extracting content from a valid chunk."""+ vlm = OpenAIVLM({"api_key": "test"})+ chunk = MockChunk(content="Hello world")+ assert vlm._extract_content_from_chunk(chunk) == "Hello world"++ def test_extract_content_from_chunk_empty(self):+ """Test extracting content from empty chunk."""+ vlm = OpenAIVLM({"api_key": "test"})+ chunk = MockChunk(content=None)+ assert vlm._extract_content_from_chunk(chunk) is None++ def test_extract_content_from_chunk_no_choices(self):+ """Test extracting content from chunk without choices."""+ vlm = OpenAIVLM({"api_key": "test"})+ chunk = MagicMock()+ chunk.choices = []+ assert vlm._extract_content_from_chunk(chunk) is None++ def test_extract_usage_from_chunk(self):+ """Test extracting usage from chunk."""+ vlm = OpenAIVLM({"api_key": "test"})+ chunk = MockChunk(usage=MockUsage(10, 20))+ pt, ct = vlm._extract_usage_from_chunk(chunk)+ assert pt == 10+ assert ct == 20++ def test_extract_usage_from_chunk_no_usage(self):+ """Test extracting usage from chunk without usage."""+ vlm = OpenAIVLM({"api_key": "test"})+ chunk = MockChunk()+ pt, ct = vlm._extract_usage_from_chunk(chunk)+ assert pt == 0+ assert ct == 0+++class TestOpenAIVLMStreamingExtraction:+ """Test streaming response extraction."""++ def test_process_streaming_chunks(self):+ """Test processing multiple streaming chunks."""+ vlm = OpenAIVLM({"api_key": "test"})++ chunks = [+ MockChunk(content="Hello"),+ MockChunk(content=" "),+ MockChunk(content="world"),+ ]++ content, pt, ct = vlm._process_streaming_chunks(chunks)+ assert content == "Hello world"+ assert pt == 0+ assert ct == 0++ def test_process_streaming_chunks_with_usage(self):+ """Test processing chunks with token usage."""+ vlm = OpenAIVLM({"api_key": "test"})++ chunks = [+ MockChunk(content="Hello", usage=MockUsage(10, 1)),+ MockChunk(content=" world", usage=MockUsage(10, 2)),+ MockChunk(content="!"),+ ]++ content, pt, ct = vlm._process_streaming_chunks(chunks)+ assert content == "Hello world!"+ # Last chunk with usage wins+ assert pt == 10+ assert ct == 2++ def test_extract_content_and_usage_streaming(self):+ """Test full extraction from streaming response."""+ vlm = OpenAIVLM({"api_key": "test"})++ chunks = [+ MockChunk(content="Streamed"),+ MockChunk(content=" content"),+ ]++ class MockStreamingResponse:+ def __iter__(self):+ return iter(chunks)++ response = MockStreamingResponse()+ content, pt, ct, is_streaming = vlm._extract_content_and_usage(response)++ assert content == "Streamed content"+ assert is_streaming is True++ def test_extract_content_and_usage_non_streaming(self):+ """Test full extraction from non-streaming response."""+ vlm = OpenAIVLM({"api_key": "test"})++ class NonStreamingResponse:+ def __init__(self):+ self.choices = [MagicMock()]+ self.choices[0].message.content = "Normal response"+ self.usage = MagicMock()+ self.usage.prompt_tokens = 10+ self.usage.completion_tokens = 5++ response = NonStreamingResponse()+ content, pt, ct, is_streaming = vlm._extract_content_and_usage(response)++ assert content == "Normal response"+ assert pt == 10+ assert ct == 5+ assert is_streaming is False+++class TestOpenAIVLMStreamingAsync:+ """Test async streaming response parsing."""++ @pytest.mark.asyncio+ async def test_extract_content_and_usage_async_streaming(self):+ """Test extracting content from async streaming response."""+ vlm = OpenAIVLM({"api_key": "test"})++ chunks = [+ MockChunk(content="Hello"),+ MockChunk(content=" async"),+ MockChunk(content=" world"),+ ]++ class MockAsyncStreamingResponse:+ def __init__(self, chunks):+ self.chunks = chunks+ self.index = 0++ def __aiter__(self):+ return self++ async def __anext__(self):+ if self.index >= len(self.chunks):+ raise StopAsyncIteration+ chunk = self.chunks[self.index]+ self.index += 1+ return chunk++ response = MockAsyncStreamingResponse(chunks)+ content, pt, ct, is_streaming = await vlm._extract_content_and_usage_async(response)++ assert content == "Hello async world"+ assert is_streaming is True++ @pytest.mark.asyncio+ async def test_extract_content_and_usage_async_non_streaming(self):+ """Test extracting content from async non-streaming response."""+ vlm = OpenAIVLM({"api_key": "test"})++ class AsyncNonStreamingResponse:+ def __init__(self):+ self.choices = [MagicMock()]+ self.choices[0].message.content = "Async response"+ self.usage = None++ response = AsyncNonStreamingResponse()+ content, pt, ct, is_streaming = await vlm._extract_content_and_usage_async(response)++ assert content == "Async response"+ assert is_streaming is False+++class TestOpenAIVLMIntegration:+ """Integration tests with mocked OpenAI client."""++ @patch("openviking.models.vlm.backends.openai_vlm.openai.OpenAI")+ def test_get_completion_with_streaming_response(self, mock_openai_class):+ """Test get_completion when API returns streaming format."""+ mock_client = MagicMock()+ mock_openai_class.return_value = mock_client++ chunks = [+ MockChunk(content="Streamed"),+ MockChunk(content=" response"),+ MockChunk(content=" text"),+ ]++ class MockStream:+ def __iter__(self):+ return iter(chunks)++ mock_client.chat.completions.create.return_value = MockStream()++ vlm = OpenAIVLM({"api_key": "test", "model": "gpt-4o-mini"})+ result = vlm.get_completion("Hello")++ assert result == "Streamed response text"++ @patch("openviking.models.vlm.backends.openai_vlm.openai.OpenAI")+ def test_get_completion_with_non_streaming_response(self, mock_openai_class):+ """Test get_completion when API returns normal format."""+ mock_client = MagicMock()+ mock_openai_class.return_value = mock_client++ class NonStreamingResponse:+ def __init__(self):+ self.choices = [MagicMock()]+ self.choices[0].message.content = "Normal response"+ self.usage = MagicMock(prompt_tokens=10, completion_tokens=5)++ mock_client.chat.completions.create.return_value = NonStreamingResponse()++ vlm = OpenAIVLM({"api_key": "test", "model": "gpt-4o-mini"})+ result = vlm.get_completion("Hello")++ assert result == "Normal response"+ # Verify token usage was updated+ usage = vlm.get_token_usage_summary()+ assert usage["total_prompt_tokens"] == 10+ assert usage["total_completion_tokens"] == 5++ @patch("openviking.models.vlm.backends.openai_vlm.openai.OpenAI")+ def test_get_vision_completion_with_streaming(self, mock_openai_class):+ """Test vision completion with streaming response."""+ mock_client = MagicMock()+ mock_openai_class.return_value = mock_client++ chunks = [+ MockChunk(content="Vision"),+ MockChunk(content=" result"),+ ]++ class MockStream:+ def __iter__(self):+ return iter(chunks)++ mock_client.chat.completions.create.return_value = MockStream()++ vlm = OpenAIVLM({"api_key": "test", "model": "gpt-4o-mini"})+ result = vlm.get_vision_completion("Describe this", ["image_url"])++ assert result == "Vision result"++ @patch("openviking.models.vlm.backends.openai_vlm.openai.OpenAI")+ def test_token_usage_with_streaming_chunks(self, mock_openai_class):+ """Test token usage extraction from streaming chunks."""+ mock_client = MagicMock()+ mock_openai_class.return_value = mock_client++ chunks = [+ MockChunk(content="Hello", usage=MockUsage(40, 1)),+ MockChunk(content=" world", usage=MockUsage(40, 2)),+ MockChunk(content="!", usage=MockUsage(40, 3)),+ ]++ class MockStream:+ def __iter__(self):+ return iter(chunks)++ mock_client.chat.completions.create.return_value = MockStream()++ vlm = OpenAIVLM({"api_key": "test", "model": "test-model"})+ result = vlm.get_completion("Hello")++ assert result == "Hello world!"+ usage = vlm.get_token_usage_summary()+ assert usage["total_prompt_tokens"] == 40+ assert usage["total_completion_tokens"] == 3