Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix GPUStack model discovery and audio support (#18599)

Javen Yanba523a127a39
diff --git a/rag/llm/model_meta.py b/rag/llm/model_meta.py
index 1bddce0198..d1857540dd 100644
--- a/rag/llm/model_meta.py
+++ b/rag/llm/model_meta.py
@@ -1080,6 +1080,10 @@ class VLLM(OpenAIAPICompatible):
_FACTORY_NAME = "VLLM"
+class GPUStack(OpenAIAPICompatible):
+ _FACTORY_NAME = "GPUStack"
+
+
class LMStudio(OpenAIAPICompatible):
_FACTORY_NAME = "LM-Studio"
diff --git a/rag/llm/rerank_model.py b/rag/llm/rerank_model.py
index 4a09c104dd..2380bb5fc1 100644
--- a/rag/llm/rerank_model.py
+++ b/rag/llm/rerank_model.py
@@ -24,11 +24,11 @@ from http import HTTPStatus
import numpy as np
import requests
-from yarl import URL
from common.log_utils import log_exception
from common.token_utils import num_tokens_from_string, truncate, total_token_count_from_response
from rag.llm.mws_utils import mws_api_url, require_mws_token
+from rag.utils.url_utils import append_api_path, ensure_v1
class Base(ABC):
@@ -642,7 +642,6 @@ class QWenRerank(Base):
_FACTORY_NAME = "Tongyi-Qianwen"
def __init__(self, key, model_name="gte-rerank-v2", **kwargs):
-
self.api_key = key
self.model_name = model_name if model_name else "gte-rerank-v2"
# Remove invalid global timeout, use official SDK per-request timeout parameter
@@ -721,6 +720,8 @@ class HuggingfaceRerank(Base):
class GPUStackRerank(Base):
+ """GPUStack reranking adapter."""
+
_FACTORY_NAME = "GPUStack"
def __init__(self, key, model_name, base_url):
@@ -728,7 +729,7 @@ class GPUStackRerank(Base):
raise ValueError("url cannot be None")
self.model_name = model_name
- self.base_url = str(URL(base_url) / "v1" / "rerank")
+ self.base_url = append_api_path(ensure_v1(base_url), "rerank")
self.headers = {
"accept": "application/json",
"content-type": "application/json",
diff --git a/rag/llm/sequence2txt_model.py b/rag/llm/sequence2txt_model.py
index 14588de794..ca3ad452cc 100644
--- a/rag/llm/sequence2txt_model.py
+++ b/rag/llm/sequence2txt_model.py
@@ -23,6 +23,7 @@ import struct
import tempfile
from abc import ABC
from collections.abc import Mapping, Sequence
+from typing import ClassVar
from urllib.parse import urlparse
import requests
@@ -515,20 +516,68 @@ class TencentCloudSeq2txt(Base):
class GPUStackSeq2txt(Base):
+ """GPUStack speech-to-text adapter."""
+
_FACTORY_NAME = "GPUStack"
+ _LANGUAGE_ALIASES: ClassVar[dict[str, str]] = {
+ "chinese": "zh",
+ "中文": "zh",
+ "zh_cn": "zh",
+ "zh-cn": "zh",
+ "english": "en",
+ "英语": "en",
+ "en-us": "en",
+ "en_us": "en",
+ }
def __init__(self, key, model_name, base_url):
if not base_url:
raise ValueError("url cannot be None")
- if base_url.split("/")[-1] != "v1":
- base_url = os.path.join(base_url, "v1")
- self.base_url = base_url
+ self.base_url = ensure_v1(base_url)
self.model_name = model_name
self.key = key
- def check_available(self) -> tuple[bool, str]:
- """GPUStack ASR transcription endpoint is not yet implemented."""
- return False, "GPUStack ASR transcription is not yet implemented"
+ def transcription(self, audio, language="Chinese", prompt=None, response_format="json"):
+ """Transcribe audio through GPUStack without logging sensitive request data."""
+ if isinstance(audio, str):
+ with open(audio, "rb") as audio_file:
+ audio_data = audio_file.read()
+ audio_file_name = os.path.basename(audio)
+ else:
+ audio_data = audio
+ audio_file_name = "audio.wav"
+
+ normalized_language = self._LANGUAGE_ALIASES.get(language.strip().lower(), language) if language else None
+ payload = {
+ "model": self.model_name,
+ "prompt": prompt,
+ "response_format": response_format,
+ }
+ if normalized_language:
+ payload["language"] = normalized_language
+ files = {"file": (audio_file_name, audio_data, "audio/wav")}
+ headers = {"Authorization": f"Bearer {self.key}"} if self.key else {}
+
+ try:
+ logger.info("GPUStack ASR request started for model %s", self.model_name)
+ response = requests.post(url=append_api_path(self.base_url, "audio/transcriptions"), files=files, data=payload, headers=headers, timeout=60)
+ response.raise_for_status()
+ result = response.json()
+ if not isinstance(result, Mapping):
+ raise ValueError("response root must be an object")
+ raw_text = result.get("text")
+ if isinstance(raw_text, str) and (text := raw_text.strip()):
+ logger.info("GPUStack ASR request completed for model %s with status %s and response text", self.model_name, response.status_code)
+ return text, num_tokens_from_string(text)
+ logger.info("GPUStack ASR request completed for model %s with status %s and empty response text", self.model_name, response.status_code)
+ return "**ERROR**: Failed to retrieve transcription.", 0
+ except requests.exceptions.RequestException as e:
+ response_status = getattr(getattr(e, "response", None), "status_code", "unknown")
+ logger.warning("GPUStack ASR request failed for model %s with status %s", self.model_name, response_status)
+ return f"**ERROR**: {e!s}", 0
+ except (TypeError, ValueError, KeyError, AttributeError) as e:
+ logger.warning("GPUStack ASR response parsing failed for model %s", self.model_name)
+ return f"**ERROR**: Invalid transcription response: {e!s}", 0
class GiteeSeq2txt(Base):
diff --git a/rag/llm/tts_model.py b/rag/llm/tts_model.py
index a850bf3b57..d478205ff0 100644
--- a/rag/llm/tts_model.py
+++ b/rag/llm/tts_model.py
@@ -19,6 +19,7 @@ import base64
import hashlib
import hmac
import json
+import logging
import os
import queue
import re
@@ -28,7 +29,6 @@ from abc import ABC
from datetime import datetime
from time import mktime
from typing import Annotated, Literal
-import logging
from urllib.parse import urlencode
from wsgiref.handlers import format_date_time
@@ -43,6 +43,8 @@ from common.token_utils import num_tokens_from_string
from rag.utils.url_utils import ensure_v1
from rag.llm.key_utils import _resolve_provider_credentials
+logger = logging.getLogger(__name__)
+
class ServeReferenceAudio(BaseModel):
audio: bytes
@@ -378,6 +380,8 @@ class OllamaTTS(HTTPBasedTTS):
class GPUStackTTS(HTTPBasedTTS):
+ """GPUStack text-to-speech adapter."""
+
_FACTORY_NAME = "GPUStack"
def __init__(self, key, model_name, **kwargs):
@@ -385,6 +389,8 @@ class GPUStackTTS(HTTPBasedTTS):
super().__init__(key, model_name, base_url)
# Add accept header
self.headers["accept"] = "application/json"
+ self.default_voice = os.environ.get("GPUSTACK_TTS_VOICE", "aiden")
+ logger.info("GPUStack TTS initialized for model %s", self.model_name)
def _process_response(self, response):
# Use chunk_size=1024 for processing response
@@ -393,10 +399,33 @@ class GPUStackTTS(HTTPBasedTTS):
yield chunk
def tts(self, text, voice="Chinese Female", stream=True):
+ """Generate speech through GPUStack without logging input text."""
text = self.normalize_text(text)
- payload = self._build_payload(text, voice)
- response = self._send_request("/v1/audio/speech", payload, stream=stream)
- return self._process_response(response)
+ voice_value = (voice or self.default_voice).strip()
+ voice_aliases = {
+ "chinese female": self.default_voice,
+ "chinese_female": self.default_voice,
+ "alloy": self.default_voice,
+ }
+ selected_voice = voice_aliases.get(voice_value.lower(), voice_value)
+ logger.info("GPUStack TTS request started for model %s with voice %s", self.model_name, selected_voice)
+ payload = self._build_payload(text, selected_voice)
+ try:
+ response = self._send_request("/audio/speech", payload, stream=stream)
+ logger.info("GPUStack TTS response received for model %s with status %s", self.model_name, response.status_code)
+ except Exception:
+ logger.warning("GPUStack TTS request failed for model %s before audio streaming", self.model_name)
+ raise
+ return self._process_response_with_logging(response)
+
+ def _process_response_with_logging(self, response):
+ """Stream GPUStack audio and log completion only after it is consumed."""
+ try:
+ yield from self._process_response(response)
+ logger.info("GPUStack TTS audio stream completed for model %s with status %s", self.model_name, response.status_code)
+ except Exception:
+ logger.warning("GPUStack TTS audio stream failed for model %s with status %s", self.model_name, getattr(response, "status_code", "unknown"))
+ raise
class SILICONFLOWTTS(HTTPBasedTTS):
diff --git a/test/unit_test/rag/llm/test_model_meta_gpustack.py b/test/unit_test/rag/llm/test_model_meta_gpustack.py
new file mode 100644
index 0000000000..84e4187ca1
--- /dev/null
+++ b/test/unit_test/rag/llm/test_model_meta_gpustack.py
@@ -0,0 +1,54 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import pytest
+
+from common.constants import LLMType
+from rag.llm.model_meta import GPUStack
+
+
+pytestmark = pytest.mark.p2
+
+
+def test_gpustack_formats_openai_compatible_model_list():
+ provider = GPUStack(api_key="sk-test", base_url="http://gpustack:80")
+
+ models = provider._format_model_list(
+ {
+ "object": "list",
+ "data": [
+ {"id": "qwen3.8-27b-fp8", "object": "model"},
+ {"id": "qwen3-embedding-0.6b", "object": "model"},
+ {"id": "qwen3-reranker-4b", "object": "model"},
+ {"id": "qwen3-asr-0.6b", "object": "model"},
+ {"id": "qwen3-tts-12hz-0.6b-customvoice", "object": "model"},
+ {"object": "model"},
+ ],
+ }
+ )
+
+ assert models == [
+ {"name": "qwen3.8-27b-fp8", "model_types": [LLMType.CHAT.value], "features": [], "max_tokens": 8192},
+ {"name": "qwen3-embedding-0.6b", "model_types": [LLMType.EMBEDDING.value], "features": [], "max_tokens": 8192},
+ {"name": "qwen3-reranker-4b", "model_types": [LLMType.RERANK.value], "features": [], "max_tokens": 8192},
+ {"name": "qwen3-asr-0.6b", "model_types": [LLMType.ASR.value], "features": [], "max_tokens": 8192},
+ {"name": "qwen3-tts-12hz-0.6b-customvoice", "model_types": [LLMType.TTS.value], "features": [], "max_tokens": 8192},
+ ]
+
+
+def test_gpustack_uses_models_endpoint_without_duplicate_v1():
+ assert GPUStack(api_key="sk-test", base_url="http://gpustack:80")._get_model_list_url() == "http://gpustack:80/v1/models"
+ assert GPUStack(api_key="sk-test", base_url="http://gpustack:80/v1")._get_model_list_url() == "http://gpustack:80/v1/models"
diff --git a/test/unit_test/rag/llm/test_rerank_gpustack.py b/test/unit_test/rag/llm/test_rerank_gpustack.py
new file mode 100644
index 0000000000..925f2534bd
--- /dev/null
+++ b/test/unit_test/rag/llm/test_rerank_gpustack.py
@@ -0,0 +1,47 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from unittest.mock import MagicMock, patch
+
+import numpy as np
+import pytest
+
+from rag.llm.rerank_model import GPUStackRerank
+
+pytestmark = pytest.mark.p2
+
+
+ ("gpustack_base_url", "expected_url"),
+ [
+ ("http://gpustack:80", "http://gpustack:80/v1/rerank"),
+ ("http://gpustack:80/v1", "http://gpustack:80/v1/rerank"),
+ ("http://gpustack:80/v1/rerank", "http://gpustack:80/v1/rerank"),
+ ],
+)
+def test_gpustack_rerank_uses_single_versioned_endpoint(gpustack_base_url, expected_url):
+ response = MagicMock()
+ response.raise_for_status.return_value = None
+ response.json.return_value = {"results": [{"index": 0, "relevance_score": 0.8}, {"index": 1, "relevance_score": 0.2}]}
+
+ with patch("rag.llm.rerank_model.requests.post", return_value=response) as post, patch("rag.llm.rerank_model.num_tokens_from_string", return_value=1):
+ rank, token_count = GPUStackRerank("sk-test", "qwen3-reranker-4b", gpustack_base_url).similarity("query", ["first", "second"])
+
+ post.assert_called_once()
+ assert post.call_args.args[0] == expected_url
+ assert post.call_args.kwargs["json"]["model"] == "qwen3-reranker-4b"
+ assert token_count == 2
+ assert np.allclose(rank, [0.8, 0.2])
diff --git a/test/unit_test/rag/llm/test_sequence2txt_gpustack.py b/test/unit_test/rag/llm/test_sequence2txt_gpustack.py
new file mode 100644
index 0000000000..eb2fb2c0c3
--- /dev/null
+++ b/test/unit_test/rag/llm/test_sequence2txt_gpustack.py
@@ -0,0 +1,91 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from rag.llm.sequence2txt_model import GPUStackSeq2txt
+
+
+pytestmark = pytest.mark.p2
+
+
+ ("gpustack_base_url", "expected_url"),
+ [
+ ("http://gpustack:80", "http://gpustack:80/v1/audio/transcriptions"),
+ ("http://gpustack:80/v1", "http://gpustack:80/v1/audio/transcriptions"),
+ ],
+)
+def test_gpustack_asr_transcription_uses_openai_compatible_endpoint(tmp_path, gpustack_base_url, expected_url):
+ audio_path = tmp_path / "sample.wav"
+ audio_path.write_bytes(b"RIFF-test-audio")
+ response = MagicMock()
+ response.json.return_value = {"text": " hello from GPUStack "}
+
+ with patch("rag.llm.sequence2txt_model.requests.post", return_value=response) as post, patch("rag.llm.sequence2txt_model.num_tokens_from_string", return_value=3):
+ provider = GPUStackSeq2txt("sk-test", "qwen3-asr", gpustack_base_url)
+ text, token_count = provider.transcription(str(audio_path))
+
+ assert text == "hello from GPUStack"
+ assert token_count == 3
+ post.assert_called_once()
+ assert post.call_args.kwargs["url"] == expected_url
+ assert post.call_args.kwargs["data"]["model"] == "qwen3-asr"
+ assert post.call_args.kwargs["data"]["language"] == "zh"
+ assert post.call_args.kwargs["headers"] == {"Authorization": "Bearer sk-test"}
+
+
+def test_gpustack_asr_check_available_transcribes_generated_wav():
+ provider = GPUStackSeq2txt("sk-test", "qwen3-asr", "http://gpustack:80/v1")
+
+ with patch.object(provider, "transcription", return_value=("hello", 1)) as transcription:
+ ok, reason = provider.check_available()
+
+ assert ok is True
+ assert reason == ""
+ transcription.assert_called_once()
+
+
+def test_gpustack_asr_returns_error_for_non_object_response(tmp_path):
+ audio_path = tmp_path / "sample.wav"
+ audio_path.write_bytes(b"RIFF-test-audio")
+ response = MagicMock()
+ response.status_code = 200
+ response.json.return_value = []
+ provider = GPUStackSeq2txt("sk-test", "qwen3-asr", "http://gpustack:80")
+
+ with patch("rag.llm.sequence2txt_model.requests.post", return_value=response):
+ text, token_count = provider.transcription(str(audio_path))
+
+ assert text.startswith("**ERROR**: Invalid transcription response:")
+ assert token_count == 0
+
+
+def test_gpustack_asr_returns_error_for_non_string_text(tmp_path):
+ audio_path = tmp_path / "sample.wav"
+ audio_path.write_bytes(b"RIFF-test-audio")
+ response = MagicMock()
+ response.status_code = 200
+ response.json.return_value = {"text": 123}
+ provider = GPUStackSeq2txt("sk-test", "qwen3-asr", "http://gpustack:80")
+
+ with patch("rag.llm.sequence2txt_model.requests.post", return_value=response):
+ text, token_count = provider.transcription(str(audio_path))
+
+ assert text == "**ERROR**: Failed to retrieve transcription."
+ assert token_count == 0
diff --git a/test/unit_test/rag/llm/test_tts_gpustack.py b/test/unit_test/rag/llm/test_tts_gpustack.py
new file mode 100644
index 0000000000..4dd076f59b
--- /dev/null
+++ b/test/unit_test/rag/llm/test_tts_gpustack.py
@@ -0,0 +1,95 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from rag.llm.tts_model import GPUStackTTS
+
+
+pytestmark = pytest.mark.p2
+
+
+def test_gpustack_tts_uses_openai_audio_speech_path_without_duplicate_v1():
+ provider = GPUStackTTS("sk-test", "qwen3-tts", base_url="http://gpustack:80")
+
+ with patch.object(provider, "_send_request") as send_request, patch.object(provider, "_process_response", return_value=iter([b"audio"])):
+ list(provider.tts("hello", voice="aiden"))
+
+ send_request.assert_called_once()
+ assert send_request.call_args.args[0] == "/audio/speech"
+ assert send_request.call_args.args[1]["voice"] == "aiden"
+
+
+def test_gpustack_tts_maps_ragflow_default_voice_to_supported_default(monkeypatch):
+ monkeypatch.setenv("GPUSTACK_TTS_VOICE", "serena")
+ provider = GPUStackTTS("sk-test", "qwen3-tts", base_url="http://gpustack:80")
+
+ with patch.object(provider, "_send_request") as send_request, patch.object(provider, "_process_response", return_value=iter([b"audio"])):
+ list(provider.tts("hello"))
+
+ assert send_request.call_args.args[1]["voice"] == "serena"
+
+
+def test_gpustack_tts_preserves_non_alias_voice_casing():
+ provider = GPUStackTTS("sk-test", "qwen3-tts", base_url="http://gpustack:80")
+
+ with patch.object(provider, "_send_request") as send_request, patch.object(provider, "_process_response", return_value=iter([b"audio"])):
+ list(provider.tts("hello", voice="Uncle_Fu"))
+
+ assert send_request.call_args.args[1]["voice"] == "Uncle_Fu"
+
+
+def test_gpustack_tts_logs_stream_completion_after_audio_consumption():
+ provider = GPUStackTTS("sk-test", "qwen3-tts", base_url="http://gpustack:80")
+ response = MagicMock()
+ response.status_code = 200
+ events = []
+
+ def audio_iterator():
+ events.append("audio-yielded")
+ yield b"audio"
+ events.append("audio-exhausted")
+
+ def record_info(message, *args):
+ events.append(message)
+
+ with (
+ patch.object(provider, "_send_request", return_value=response),
+ patch.object(provider, "_process_response", return_value=audio_iterator()),
+ patch("rag.llm.tts_model.logger.info", side_effect=record_info),
+ ):
+ stream = provider.tts("hello", voice="aiden")
+ assert "GPUStack TTS response received for model %s with status %s" in events
+ assert "GPUStack TTS audio stream completed for model %s with status %s" not in events
+ assert next(stream) == b"audio"
+ assert "GPUStack TTS audio stream completed for model %s with status %s" not in events
+ with pytest.raises(StopIteration):
+ next(stream)
+
+ assert events.index("audio-exhausted") < events.index("GPUStack TTS audio stream completed for model %s with status %s")
+
+
+def test_gpustack_tts_request_failure_log_omits_exception_text():
+ provider = GPUStackTTS("sk-test", "qwen3-tts", base_url="http://gpustack:80")
+
+ with patch.object(provider, "_send_request", side_effect=RuntimeError("secret response body")), patch("rag.llm.tts_model.logger.warning") as warning, pytest.raises(RuntimeError):
+ list(provider.tts("hello", voice="aiden"))
+
+ warning.assert_called_once_with("GPUStack TTS request failed for model %s before audio streaming", "qwen3-tts")
+ logged_parts = " ".join(str(arg) for call in warning.call_args_list for arg in call.args)
+ assert "secret response body" not in logged_parts