Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(chat): images uploaded in chat UI were ignored by vision models (agentic loop / Go attachment drop) (#18884)

euvre2af732d6072f
diff --git a/api/db/services/dialog_service.py b/api/db/services/dialog_service.py
index ad1c8c5c13..d6c607568e 100644
--- a/api/db/services/dialog_service.py
+++ b/api/db/services/dialog_service.py
@@ -35,6 +35,7 @@ from api.db.services.doc_metadata_service import DocMetadataService
from api.db.services.knowledgebase_service import KnowledgebaseService, validate_dataset_embedding_models
from api.db.services.langfuse_service import TenantLangfuseService
from api.db.services.llm_service import LLMBundle, resolve_llm_setting
+from api.db.services.user_service import TenantService
from common.metadata_utils import apply_meta_data_filter
from api.utils.reference_metadata_utils import (
enrich_chunks_with_document_metadata,
@@ -582,6 +583,14 @@ def repair_bad_citation_formats(answer: str, kbinfos: dict, idx: set):
return answer, idx
+def _empty_response_applies(knowledges: list, text_attachments_content: str, image_attachments: list, image_files: list) -> bool:
+ """The configured empty-response fallback applies only when retrieval
+ found nothing AND the message carries no attachment context (text or
+ images) the model could still answer from — an attached image must
+ reach the model instead of being swallowed by the canned response."""
+ return not knowledges and not text_attachments_content and not image_attachments and not image_files
+
+
async def async_chat(dialog, messages, stream=True, **kwargs):
logging.debug("Begin async_chat")
assert messages[-1]["role"] == "user", "The last content of this conversation is not from user."
@@ -791,7 +800,10 @@ async def async_chat(dialog, messages, stream=True, **kwargs):
logging.debug("{}->{}".format(" ".join(questions), "\n->".join(knowledges)))
retrieval_ts = timer()
- if not knowledges and prompt_config.get("empty_response"):
+ # Attachments (text or images) are context the model can still answer
+ # from, so the configured fallback must not eat the request before the
+ # images reach the model.
+ if _empty_response_applies(knowledges, text_attachments_content, image_attachments, image_files) and prompt_config.get("empty_response"):
empty_res = prompt_config["empty_response"]
logging.debug("async_chat empty_response path: empty_res=%r tts_mdl=%r", empty_res, tts_mdl)
# HTML-escape for frontend display so DOMPurify does not strip
@@ -827,7 +839,13 @@ async def async_chat(dialog, messages, stream=True, **kwargs):
msg[-1]["content"] += text_attachments_content
used_token_count, msg = message_fit_in(msg, int(max_tokens * 0.95))
if llm_model_config["model_type"] == "chat" and image_attachments:
- convert_last_user_msg_to_multimodal(msg, image_attachments, factory)
+ if dialog_model_vision_capable(dialog):
+ convert_last_user_msg_to_multimodal(msg, image_attachments, factory)
+ else:
+ # Text-only chat models reject image content blocks at the
+ # provider (e.g. Zhipu GLM error 1210: messages.content.type
+ # only allows 'text'); answer from the text alone instead.
+ logging.info("async_chat: dropping image attachments for text-only chat model")
assert len(msg) >= 2, f"message_fit_in has bug: {msg}"
prompt = msg[0]["content"]
@@ -1950,6 +1968,75 @@ def _render_reasoning_system_prompt(dialog, prompt_config: dict, kwargs: dict) -
return system.format(**fmt_kwargs)
+def message_has_image_attachments(message: dict) -> bool:
+ """True when the message carries at least one image attachment.
+
+ Covers both shapes the web client sends: file dicts with a ``mime_type``
+ produced by the upload flow, and legacy pre-resolved ``data:`` URI
+ strings. The detection mirrors the fetch path — ``FileService.get_files``
+ classifies images by a substring ``image`` match on ``mime_type`` and
+ ``split_file_attachments`` treats every ``data:`` URI as an image — so
+ the routing decision never disagrees with what the pipeline fetches.
+ """
+ files = message.get("files") or []
+ if isinstance(files, dict):
+ files = [files]
+ if not isinstance(files, list):
+ return False
+ for f in files:
+ if isinstance(f, dict):
+ if "image" in str(f.get("mime_type") or ""):
+ return True
+ elif isinstance(f, str) and f.strip().startswith("data:"):
+ return True
+ return False
+
+
+def _resolve_dialog_chat_model_types(dialog) -> list[str]:
+ """Enrolled model types of the chat model the dialog will actually use.
+
+ Mirrors the chat-model resolution in ``get_models``/``async_chat``: a
+ tenant-level override that can serve as a chat model wins over the
+ dialog's ``llm_id``, and an empty ``llm_id`` falls back to the tenant's
+ default chat model. Unresolvable references yield an empty list.
+ """
+ tenant_id = getattr(dialog, "tenant_id", "")
+ llm_id = getattr(dialog, "llm_id", "") or ""
+ tenant_llm_id = getattr(dialog, "tenant_llm_id", "") or ""
+ if llm_id:
+ try:
+ llm_types = resolve_model_type(tenant_id, llm_id)
+ except LookupError:
+ llm_types = []
+ if tenant_llm_id:
+ try:
+ override_types = resolve_model_type(tenant_id, tenant_llm_id)
+ except LookupError:
+ override_types = []
+ # The override replaces the chat model only when it is enrolled
+ # as one (the same check get_model_config_by_id applies).
+ if "chat" in override_types:
+ return override_types
+ return llm_types
+ exist, tenant = TenantService.get_by_id(tenant_id)
+ if not exist:
+ return []
+ default_ref = (getattr(tenant, "tenant_llm_id", "") or "") or getattr(tenant, "llm_id", "")
+ if not default_ref:
+ return []
+ try:
+ return resolve_model_type(tenant_id, default_ref)
+ except LookupError:
+ return []
+
+
+def dialog_model_vision_capable(dialog) -> bool:
+ """True when the chat model the dialog will use is enrolled as
+ vision-capable (``ModelTypeBinary.VISION`` bit). Unresolvable models are
+ conservatively treated as text-only."""
+ return "vision" in _resolve_dialog_chat_model_types(dialog)
+
+
async def rag_agent(dialog, messages, stream=True, **kwargs):
prompt_config = dialog.prompt_config or {}
assert messages[-1]["role"] == "user", "The last content of this conversation is not from user."
@@ -1958,6 +2045,17 @@ async def rag_agent(dialog, messages, stream=True, **kwargs):
async for ans in async_chat(dialog, messages, stream, **kwargs):
yield ans
return
+ # The agentic loop below composes the final answer from retrieval results
+ # only: the terminal `rag` tool streams the inner graph's cited answer, so
+ # an image attached to the question can never influence it. When the chat
+ # model is vision-capable and the user attached images, answer through
+ # async_chat instead — it embeds the images into the model messages and
+ # still retrieves from the KBs and adds citations.
+ if message_has_image_attachments(messages[-1]) and dialog_model_vision_capable(dialog):
+ logging.info("rag_agent: vision-capable model with image attachments; routing to async_chat so the model sees the images")
+ async for ans in async_chat(dialog, messages, stream, **kwargs):
+ yield ans
+ return
kbs, embd_mdl, rerank_mdl, chat_mdl, tts_mdl = get_models(dialog)
# Agentic RAG depends on the outer model being able to call the bound
@@ -1981,7 +2079,13 @@ async def rag_agent(dialog, messages, stream=True, **kwargs):
if text_attachments_content and agent_messages:
agent_messages[-1]["content"] += text_attachments_content
if model_type == "chat" and image_attachments:
- convert_last_user_msg_to_multimodal(agent_messages, image_attachments, factory)
+ if dialog_model_vision_capable(dialog):
+ convert_last_user_msg_to_multimodal(agent_messages, image_attachments, factory)
+ else:
+ # Text-only chat models reject image content blocks at the
+ # provider (e.g. Zhipu GLM error 1210: messages.content.type
+ # only allows 'text'); answer from the text alone instead.
+ logging.info("rag_agent: dropping image attachments for text-only chat model")
use_web_search = _should_use_web_search(prompt_config, kwargs.get("internet"))
logging.debug("web_search kb=%s configured=%s internet=%r enabled=%s", bool(dialog.kb_ids), has_web_search_provider(prompt_config), kwargs.get("internet"), use_web_search)
tenant_ids = list(set([kb.tenant_id for kb in kbs]))
diff --git a/internal/service/chat_pipeline.go b/internal/service/chat_pipeline.go
index 799c4d7124..8037cbbc0f 100644
--- a/internal/service/chat_pipeline.go
+++ b/internal/service/chat_pipeline.go
@@ -321,17 +321,21 @@ func (s *ChatPipelineService) AsyncChat(
// Parse file attachments from the last message.
// Split text-file URLs (joined with "\n\n") and image URLs.
- // Vision (image2text) model: images → imageFiles for the multimodal
- // conversion below. Text-only chat model: images are dropped —
- // sending image content blocks makes such providers reject the
- // request (e.g. Zhipu GLM error 1210: messages.content.type only
- // allows 'text').
+ // Only vision-capable (image2text-typed) models receive image
+ // content blocks; text-only chat providers reject them (e.g.
+ // Zhipu GLM error 1210: messages.content.type only allows
+ // 'text'), so their images are dropped here — mirroring
+ // Python's dialog_model_vision_capable gate.
var textAttachmentsList []string
var imageFiles []string
+ // Whether the message carried image attachments before the
+ // vision gate (read by the empty-response fallback below).
+ var hasImageAttachments bool
// Joined text attachments (appended to system prompt).
var attachments string
// When files are file dicts, splitFileAttachments fetches blobs
- // from storage. When plain strings, falls back to string splitting.
+ // from storage. When plain strings, falls back to string splitting;
+ // raw only changes that split for pre-separated image payloads.
if files, hasFiles := lastMsg["files"]; hasFiles {
modelType := "chat"
if llmModelConfig != nil {
@@ -339,22 +343,16 @@ func (s *ChatPipelineService) AsyncChat(
modelType = mt
}
}
- if modelType == "image2text" {
- textAttachmentsList, imageFiles = splitFileAttachments(ctx, userID, files, true)
- } else {
- var droppedImages []string
- textAttachmentsList, droppedImages = splitFileAttachments(ctx, userID, files, false)
- if len(droppedImages) > 0 {
- common.Warn("AsyncChat: dropping image attachments for text-only chat model",
- zap.String("llm_id", chat.LLMID),
- zap.Int("dropped_images", len(droppedImages)))
- }
- }
+ var images []string
+ textAttachmentsList, images = splitFileAttachments(ctx, userID, files, modelType == "image2text")
+ imageFiles, hasImageAttachments = gateImageAttachments(chat.LLMID, modelType, images)
attachments = strings.Join(textAttachmentsList, "\n\n")
+ // Log counts only: attachment payloads are base64 images or
+ // full document contents.
common.Debug("Resolved attachments",
- zap.Strings("text_attachments_list", textAttachmentsList),
- zap.Strings("image_files", imageFiles),
- zap.String("attachments", attachments))
+ zap.Int("text_attachments", len(textAttachmentsList)),
+ zap.Int("image_files", len(imageFiles)),
+ zap.Int("attachment_bytes", len(attachments)))
}
// === Phase 6: SQL Retrieval ===
@@ -842,9 +840,11 @@ func (s *ChatPipelineService) AsyncChat(
//
// EXCEPTION: when the user attached files to their message, the
// attachment text provides context that should be sent to the LLM
- // even if KB retrieval returned nothing. In that case we skip the
- // early return and fall through to the normal LLM call where
- // attachments are appended to the system prompt.
+ // even if KB retrieval returned nothing — and image attachments
+ // must reach a vision model instead of being swallowed by the
+ // canned response. In that case we skip the early return and fall
+ // through to the normal LLM call where attachments are appended to
+ // the system prompt.
//
// Two results are yielded (mirroring Python dialog_service.py):
// 1. Final=false — carries the answer text so streaming consumers
@@ -854,7 +854,7 @@ func (s *ChatPipelineService) AsyncChat(
// final event (dialog_service.py:807); consumers that only look at
// the final event (e.g. the OpenAI-compatible endpoint) would
// otherwise see an empty reply.
- if len(knowledges) == 0 && attachments == "" {
+ if emptyResponseApplies(len(knowledges), attachments, hasImageAttachments) {
if emptyResp, ok := promptConfig["empty_response"].(string); ok && emptyResp != "" {
out <- AsyncChatResult{
Answer: emptyResp,
@@ -949,9 +949,9 @@ func (s *ChatPipelineService) AsyncChat(
zap.Int("used_token_count", usedTokenCount),
zap.Int("msg_count", len(llmMessages)))
- // Multimodal conversion. imageFiles is only populated for
- // vision-capable (image2text) models; text-only chat models never
- // reach this with images.
+ // Multimodal conversion. imageFiles only survives the vision gate
+ // above, so only vision-capable (image2text) models reach this
+ // with images.
if len(llmMessages) >= 2 && len(imageFiles) > 0 {
lastIdx := len(llmMessages) - 1
if role, _ := llmMessages[lastIdx]["role"].(string); role == "user" {
@@ -1352,8 +1352,8 @@ func (s *ChatPipelineService) AsyncChatSolo(
}
// 2. Process file attachments. Only vision-capable (image2text)
- // models receive image content; text-only chat models reject image
- // blocks at the provider (e.g. Zhipu GLM error 1210:
+ // models receive image content; text-only chat models reject
+ // image blocks at the provider (e.g. Zhipu GLM error 1210:
// messages.content.type only allows 'text'), so their image
// attachments are dropped here.
attachmentsStr := ""
@@ -1369,12 +1369,7 @@ func (s *ChatPipelineService) AsyncChatSolo(
if files, hasFiles := messages[len(messages)-1]["files"]; hasFiles {
var images []string
attachmentsStr, images = s.splitChatAttachments(ctx, userID, files)
- if isImage2Text {
- imageFiles = images
- } else {
- common.Debug("AsyncChatSolo: dropping image attachments for text-only chat model",
- zap.String("llm_id", chat.LLMID))
- }
+ imageFiles, _ = gateImageAttachments(chat.LLMID, modelType, images)
common.Info("AsyncChatSolo: file attachments resolved",
zap.Bool("vision_model", isImage2Text),
zap.Int("image_files", len(imageFiles)),
@@ -2093,6 +2088,34 @@ func lastUserQuestion(messages []map[string]interface{}) string {
return ""
}
+// gateImageAttachments applies the vision-capability policy to split
+// attachments: images survive only for vision-capable (image2text-typed)
+// models — text-only chat providers reject image content blocks (e.g.
+// Zhipu GLM error 1210: messages.content.type only allows 'text'), so
+// their images are dropped with a warning instead of failing the whole
+// request. It returns the surviving images and whether any image was
+// attached before the drop, so callers can still route image-bearing
+// questions past the empty-response fallback. Mirrors Python's
+// dialog_model_vision_capable gate in dialog_service.py.
+func gateImageAttachments(llmID, modelType string, images []string) (kept []string, attached bool) {
+ attached = len(images) > 0
+ if attached && modelType != "image2text" {
+ common.Warn("dropping image attachments for text-only chat model",
+ zap.String("llm_id", llmID),
+ zap.Int("dropped_images", len(images)))
+ return nil, attached
+ }
+ return images, attached
+}
+
+// emptyResponseApplies reports whether the configured empty-response
+// fallback should short-circuit the LLM call: only when retrieval found
+// nothing AND the message carries no attachment context (text or images)
+// the model could still answer from.
+func emptyResponseApplies(knowledgeCount int, attachments string, imageAttachments bool) bool {
+ return knowledgeCount == 0 && attachments == "" && !imageAttachments
+}
+
// splitFileAttachments mirrors Python's `split_file_attachments` at
// dialog_service.py:371-392. It separates `messages[-1]["files"]`
// into text-file content and image attachments.
diff --git a/internal/service/chat_pipeline_image_test.go b/internal/service/chat_pipeline_image_test.go
index 6aa7d479b7..5038d2a2f5 100644
--- a/internal/service/chat_pipeline_image_test.go
+++ b/internal/service/chat_pipeline_image_test.go
@@ -168,3 +168,72 @@ func TestSplitFileAttachmentsRawImagesFeedMultimodalConversion(t *testing.T) {
t.Fatalf("no image part in converted content: %v", rendered)
}
}
+
+// Regression (KB chat path, "chat"-typed model config): a text-only chat
+// model must not receive image content parts — providers reject them (e.g.
+// Zhipu GLM error 1210: messages.content.type only allows 'text'). The
+// vision gate drops the images after the split, so the multimodal
+// conversion never runs and the user gets a graceful text-only answer
+// instead of a provider-side rejection.
+func TestGateImageAttachmentsDropsImagesForTextOnlyChatModel(t *testing.T) {
+ seedChatImageStorage(t)
+
+ texts, images := splitFileAttachments(t.Context(), "user-1", []interface{}{chatImageFileDict}, false)
+ if len(images) != 1 {
+ t.Fatalf("split still resolves file-dict images with raw=false, got %v", images)
+ }
+ kept, attached := gateImageAttachments("glm-4-flash@Zhipu", "chat", images)
+ if len(kept) != 0 {
+ t.Fatalf("text-only chat model must not keep images, got %v", kept)
+ }
+ if !attached {
+ t.Fatal("attachment flag must report the pre-drop image so the empty-response fallback stays out of the way")
+ }
+ if len(texts) != 0 {
+ t.Fatalf("expected no text attachments for an image-only upload, got %v", texts)
+ }
+}
+
+// Vision-capable (image2text) models keep every split image, and both
+// split modes resolve file-dict attachments to jpeg data URIs that feed
+// ConvertLastUserMsgToMultimodal.
+func TestGateImageAttachmentsKeepsImagesForVisionModel(t *testing.T) {
+ seedChatImageStorage(t)
+
+ for _, raw := range []bool{false, true} {
+ texts, images := splitFileAttachments(t.Context(), "user-1", []interface{}{chatImageFileDict}, raw)
+ kept, attached := gateImageAttachments("qwen3-vl-plus@Tongyi-Qianwen", "image2text", images)
+ if !attached || len(kept) != 1 {
+ t.Fatalf("raw=%v: vision model must keep the image (kept=%v attached=%v)", raw, kept, attached)
+ }
+ if !strings.HasPrefix(kept[0], "data:image/jpeg;base64,") {
+ t.Fatalf("raw=%v: image not a jpeg data URI: %q", raw, kept[0])
+ }
+ if len(texts) != 0 {
+ t.Fatalf("raw=%v: expected no text attachments, got %v", raw, texts)
+ }
+ }
+}
+
+// The empty-response fallback must not short-circuit image-only requests:
+// the model is the only consumer of the image, so the canned response
+// would swallow it before the multimodal conversion.
+func TestEmptyResponseAppliesRequiresNoAttachments(t *testing.T) {
+ if !emptyResponseApplies(0, "", false) {
+ t.Fatal("no knowledge and no attachments: the fallback applies")
+ }
+ for _, tc := range []struct {
+ name string
+ knowledges int
+ attachments string
+ images bool
+ }{
+ {"retrieval found something", 1, "", false},
+ {"text attachments present", 0, "doc text", false},
+ {"image-only request", 0, "", true},
+ } {
+ if emptyResponseApplies(tc.knowledges, tc.attachments, tc.images) {
+ t.Fatalf("%s: fallback must not apply", tc.name)
+ }
+ }
+}
diff --git a/test/unit_test/api/db/services/test_dialog_service_rag_agent_image_routing.py b/test/unit_test/api/db/services/test_dialog_service_rag_agent_image_routing.py
new file mode 100644
index 0000000000..8805a59292
--- /dev/null
+++ b/test/unit_test/api/db/services/test_dialog_service_rag_agent_image_routing.py
@@ -0,0 +1,288 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+Regression tests for rag_agent() routing image attachments away from the
+agentic reasoning loop.
+
+The agentic loop composes its final answer from retrieval results only (the
+terminal `rag` tool streams the inner graph's cited answer), so an image
+attached to the question could never influence it: the user got the
+knowledge-base "not found" response instead of a description of the image.
+When the dialog's chat model is vision-capable and the last user message
+carries image attachments, rag_agent() must answer through async_chat(),
+which embeds the images into the model messages.
+"""
+
+import asyncio
+import sys
+import types
+import warnings
+from types import SimpleNamespace
+
+import pytest
+
+warnings.filterwarnings(
+ "ignore",
+ message="pkg_resources is deprecated as an API.*",
+ category=UserWarning,
+)
+
+
+def _install_cv2_stub_if_unavailable():
+ try:
+ import cv2 # noqa: F401
+
+ return
+ except Exception:
+ pass
+ stub = types.ModuleType("cv2")
+ stub.INTER_LINEAR = 1
+ stub.INTER_CUBIC = 2
+ stub.BORDER_CONSTANT = 0
+ stub.BORDER_REPLICATE = 1
+ stub.COLOR_BGR2RGB = 0
+ stub.COLOR_BGR2GRAY = 1
+ stub.COLOR_GRAY2BGR = 2
+ stub.IMREAD_IGNORE_ORIENTATION = 128
+ stub.IMREAD_COLOR = 1
+ stub.RETR_LIST = 1
+ stub.CHAIN_APPROX_SIMPLE = 2
+
+ def _module_getattr(name):
+ if name.isupper():
+ return 0
+ raise RuntimeError(f"cv2.{name} is unavailable in this test environment")
+
+ stub.__getattr__ = _module_getattr
+ sys.modules["cv2"] = stub
+
+
+_install_cv2_stub_if_unavailable()
+
+from api.db.services import dialog_service # noqa: E402
+from test_dialog_service_rag_agent_messages import ( # noqa: E402
+ _KB,
+ _RecordingChatModel,
+ _StubRAGTools,
+)
+
+
+_IMAGE_FILE_DICT = {"id": "img-1", "name": "photo.jpeg", "mime_type": "image/jpeg", "created_by": "tenant-1"}
+
+
+class _RecordingAsyncChat:
+ """Stands in for async_chat and records that the plain path was taken."""
+
+ def __init__(self):
+ self.calls = []
+
+ async def __call__(self, dialog, messages, stream=True, **kwargs):
+ self.calls.append((dialog, messages, stream, kwargs))
+ yield {"answer": "described the image", "reference": {}, "final": True}
+
+
+def _dialog(llm_id="qwen3-vl-plus@Tongyi-Qianwen"):
+ return SimpleNamespace(
+ id="dialog-1",
+ tenant_id="tenant-1",
+ kb_ids=["kb-1"],
+ llm_id=llm_id,
+ llm_setting={"temperature": 0.1},
+ prompt_config={},
+ meta_data_filter=None,
+ )
+
+
+def _collect(gen):
+ async def _run():
+ return [item async for item in gen]
+
+ return asyncio.run(_run())
+
+
+def _patch_agentic_stubs(monkeypatch):
+ chat_mdl = _RecordingChatModel()
+ monkeypatch.setattr(dialog_service, "get_models", lambda _dialog, **_kw: ([_KB], None, None, chat_mdl, None))
+ monkeypatch.setattr(dialog_service, "RAGTools", _StubRAGTools)
+ return chat_mdl
+
+
+def test_rag_agent_routes_vision_image_questions_to_async_chat(monkeypatch):
+ """Reasoning on + image attachment + vision-capable model must bypass the
+ agentic loop (whose answer is retrieval-only) and use async_chat."""
+ recording_chat = _RecordingAsyncChat()
+ monkeypatch.setattr(dialog_service, "async_chat", recording_chat)
+
+ def _fail_get_models(*_args, **_kwargs):
+ raise AssertionError("agentic path must not run for vision image questions")
+
+ monkeypatch.setattr(dialog_service, "get_models", _fail_get_models)
+ monkeypatch.setattr(dialog_service, "resolve_model_type", lambda _t, _m: ["chat", "vision"])
+
+ messages = [{"role": "user", "content": "描述图片内容", "id": "m-1", "files": [dict(_IMAGE_FILE_DICT)]}]
+ dialog = _dialog()
+
+ results = _collect(dialog_service.rag_agent(dialog, messages, True, reasoning=1))
+
+ assert len(recording_chat.calls) == 1
+ called_dialog, called_messages, _stream, kwargs = recording_chat.calls[0]
+ assert called_dialog is dialog
+ assert called_messages is messages
+ assert kwargs.get("reasoning") == 1
+ assert results[-1]["answer"] == "described the image"
+
+
+def test_rag_agent_keeps_agentic_loop_for_text_only_model_with_images(monkeypatch):
+ """Images attached to a text-only model cannot be answered either way, so
+ the agentic loop must keep running — and must not inject image content
+ parts the provider would reject (e.g. Zhipu GLM error 1210:
+ messages.content.type only allows 'text')."""
+ recording_chat = _RecordingAsyncChat()
+ monkeypatch.setattr(dialog_service, "async_chat", recording_chat)
+ monkeypatch.setattr(dialog_service, "resolve_model_type", lambda _t, _m: ["chat"])
+ chat_mdl = _patch_agentic_stubs(monkeypatch)
+ monkeypatch.setattr(
+ dialog_service,
+ "get_files_content",
+ lambda _last_message, _model_type: ("", ["data:image/jpeg;base64,QUJD"], []),
+ )
+ conversions = []
+ monkeypatch.setattr(
+ dialog_service,
+ "convert_last_user_msg_to_multimodal",
+ lambda *_args: conversions.append(_args),
+ )
+
+ messages = [{"role": "user", "content": "describe the image", "id": "m-1", "files": [dict(_IMAGE_FILE_DICT)]}]
+
+ results = _collect(dialog_service.rag_agent(_dialog("glm-4-flash@Zhipu"), messages, False, reasoning=1))
+
+ assert recording_chat.calls == [], "agentic path must not delegate to async_chat for text-only models"
+ assert results, "agentic path must produce an answer"
+ assert conversions == [], "text-only models must not receive image content parts"
+ assert isinstance(chat_mdl.sent_messages[-1]["content"], str)
+
+
+def test_rag_agent_keeps_agentic_loop_without_images(monkeypatch):
+ """Pure text questions with reasoning keep the agentic research loop."""
+ recording_chat = _RecordingAsyncChat()
+ monkeypatch.setattr(dialog_service, "async_chat", recording_chat)
+ monkeypatch.setattr(dialog_service, "resolve_model_type", lambda _t, _m: ["chat", "vision"])
+ _patch_agentic_stubs(monkeypatch)
+
+ messages = [{"role": "user", "content": "What is RAGFlow?", "id": "m-1"}]
+
+ results = _collect(dialog_service.rag_agent(_dialog(), messages, False, reasoning=1))
+
+ assert recording_chat.calls == []
+ assert results
+
+
+def test_message_has_image_attachments_covers_client_shapes():
+ assert dialog_service.message_has_image_attachments({"files": [_IMAGE_FILE_DICT]})
+ assert dialog_service.message_has_image_attachments({"files": ["data:image/png;base64,AAA"]})
+ assert dialog_service.message_has_image_attachments({"files": {"mime_type": "image/png"}})
+ # Aligned with the fetch path (FileService.get_files / split_file_attachments):
+ # any data: URI, and any mime_type containing "image".
+ assert dialog_service.message_has_image_attachments({"files": ["data:application/octet-stream;base64,AAA"]})
+ assert dialog_service.message_has_image_attachments({"files": [{"mime_type": "application/x-image"}]})
+ assert not dialog_service.message_has_image_attachments({"files": [{"mime_type": "application/pdf"}]})
+ assert not dialog_service.message_has_image_attachments({"files": ["https://example.com/a.png"]})
+ assert not dialog_service.message_has_image_attachments({})
+ assert not dialog_service.message_has_image_attachments({"files": "not-a-list"})
+
+
+def test_dialog_model_vision_capable_resolves_enrolled_types(monkeypatch):
+ monkeypatch.setattr(dialog_service, "resolve_model_type", lambda _t, _m: ["chat", "vision"])
+ assert dialog_service.dialog_model_vision_capable(_dialog()) is True
+
+ monkeypatch.setattr(dialog_service, "resolve_model_type", lambda _t, _m: ["chat"])
+ assert dialog_service.dialog_model_vision_capable(_dialog()) is False
+
+ def _raise(_t, _m):
+ raise LookupError("no such model")
+
+ monkeypatch.setattr(dialog_service, "resolve_model_type", _raise)
+ assert dialog_service.dialog_model_vision_capable(_dialog()) is False
+
+ # Empty llm_id falls back to the tenant default chat model — that path
+ # queries the tenant row, so stub TenantService instead of requiring a
+ # live database (same pattern as the effective-chat-model test below).
+ monkeypatch.setattr(dialog_service, "TenantService", SimpleNamespace(get_by_id=lambda _tid: (False, None)))
+ assert dialog_service.dialog_model_vision_capable(SimpleNamespace(tenant_id="t1", llm_id="")) is False
+
+
+def test_dialog_model_vision_capable_resolves_the_effective_chat_model(monkeypatch):
+ """The capability probe must follow the same chat-model resolution
+ get_models()/async_chat() use: a tenant-level override that can serve as
+ a chat model wins, and an empty llm_id falls back to the tenant default."""
+ types_by_ref = {
+ "qwen3-vl-plus@Tongyi-Qianwen": ["chat", "vision"],
+ "chat-override-id": ["chat", "vision"],
+ "text-only@Zhipu": ["chat"],
+ "vision-only-override-id": ["vision"],
+ "default-chat-model-id": ["chat", "vision"],
+ }
+
+ def _fake_resolve(_tenant, ref):
+ if ref not in types_by_ref:
+ raise LookupError(ref)
+ return types_by_ref[ref]
+
+ monkeypatch.setattr(dialog_service, "resolve_model_type", _fake_resolve)
+
+ # A chat-capable tenant override replaces the dialog's text-only model.
+ dialog = SimpleNamespace(tenant_id="tenant-1", llm_id="text-only@Zhipu", tenant_llm_id="chat-override-id")
+ assert dialog_service.dialog_model_vision_capable(dialog) is True
+
+ # A vision-only override cannot serve as the chat model: fall back to llm_id.
+ dialog = SimpleNamespace(tenant_id="tenant-1", llm_id="text-only@Zhipu", tenant_llm_id="vision-only-override-id")
+ assert dialog_service.dialog_model_vision_capable(dialog) is False
+ dialog = SimpleNamespace(tenant_id="tenant-1", llm_id="qwen3-vl-plus@Tongyi-Qianwen", tenant_llm_id="vision-only-override-id")
+ assert dialog_service.dialog_model_vision_capable(dialog) is True
+
+ # An unresolvable override falls back to llm_id.
+ dialog = SimpleNamespace(tenant_id="tenant-1", llm_id="qwen3-vl-plus@Tongyi-Qianwen", tenant_llm_id="missing-id")
+ assert dialog_service.dialog_model_vision_capable(dialog) is True
+
+ # Empty llm_id: the tenant's default chat model decides.
+ dialog = SimpleNamespace(tenant_id="tenant-1", llm_id="", tenant_llm_id="")
+ fake_tenant = SimpleNamespace(tenant_llm_id="default-chat-model-id", llm_id="")
+ monkeypatch.setattr(dialog_service, "TenantService", SimpleNamespace(get_by_id=lambda _tid: (True, fake_tenant)))
+ assert dialog_service.dialog_model_vision_capable(dialog) is True
+
+ fake_tenant = SimpleNamespace(tenant_llm_id="", llm_id="text-only@Zhipu")
+ monkeypatch.setattr(dialog_service, "TenantService", SimpleNamespace(get_by_id=lambda _tid: (True, fake_tenant)))
+ assert dialog_service.dialog_model_vision_capable(dialog) is False
+
+ monkeypatch.setattr(dialog_service, "TenantService", SimpleNamespace(get_by_id=lambda _tid: (False, None)))
+ assert dialog_service.dialog_model_vision_capable(dialog) is False
+
+
+def test_empty_response_applies_only_without_attachment_context():
+ applies = dialog_service._empty_response_applies
+ assert applies([], "", [], [])
+ assert not applies(["chunk"], "", [], [])
+ assert not applies([], "parsed document text", [], [])
+ assert not applies([], "", ["data:image/png;base64,AAA"], [])
+ assert not applies([], "", [], [b"raw-image-bytes"])
diff --git a/test/unit_test/api/db/services/test_gaussdb_dialog_sql.py b/test/unit_test/api/db/services/test_gaussdb_dialog_sql.py
index aad44677da..5fc085ae8c 100644
--- a/test/unit_test/api/db/services/test_gaussdb_dialog_sql.py
+++ b/test/unit_test/api/db/services/test_gaussdb_dialog_sql.py
@@ -207,7 +207,7 @@ def _install_settings_import_stubs(monkeypatch):
def atomic():
return _Context()
- install_module("api.db.services.user_service", UserService=_Dummy)
+ install_module("api.db.services.user_service", UserService=_Dummy, TenantService=_Dummy)
install_module("api.db.services.file_service", FileService=_Dummy)
install_module("api.db.services.common_service", CommonService=_Dummy)
install_module("api.db.services.doc_metadata_service", DocMetadataService=_Dummy)