Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(agent): reject unsupported image input cleanly (#19539)

Sbaaoui Idriss815639482c2a
diff --git a/internal/agent/canvas/run_error.go b/internal/agent/canvas/run_error.go
index 3f1b3367e6..8fa2e365a4 100644
--- a/internal/agent/canvas/run_error.go
+++ b/internal/agent/canvas/run_error.go
@@ -14,7 +14,11 @@
package canvas
-import "errors"
+import (
+ "errors"
+
+ "ragflow/internal/agent/runtime"
+)
const (
// RunErrorKindInternal marks an error whose cause is safe for server logs
@@ -48,6 +52,10 @@ func (e *internalRunError) Unwrap() error {
}
func runErrorEvent(err error) ErrorEvent {
+ var userErr *runtime.UserFacingError
+ if errors.As(err, &userErr) {
+ return ErrorEvent{Message: userErr.Error(), Kind: "user"}
+ }
var internalErr *internalRunError
if errors.As(err, &internalErr) {
return ErrorEvent{
diff --git a/internal/agent/canvas/run_error_test.go b/internal/agent/canvas/run_error_test.go
index 61ec997c73..8e9a76f2d4 100644
--- a/internal/agent/canvas/run_error_test.go
+++ b/internal/agent/canvas/run_error_test.go
@@ -21,6 +21,8 @@ import (
"fmt"
"strings"
"testing"
+
+ "ragflow/internal/agent/runtime"
)
func TestRunnerRedactsInternalRunError(t *testing.T) {
@@ -69,3 +71,13 @@ func TestRunErrorEventPreservesPublicMessage(t *testing.T) {
t.Errorf("kind = %q, want empty for public runtime error", payload.Kind)
}
}
+
+func TestRunErrorEventPreservesUserFacingMessage(t *testing.T) {
+ payload := runErrorEvent(runtime.NewUserFacingError("Image input is not supported by the selected model \"glm-4.7\". Please select a vision-capable model."))
+ if payload.Message != "Image input is not supported by the selected model \"glm-4.7\". Please select a vision-capable model." {
+ t.Errorf("message = %q, want clean user-facing message", payload.Message)
+ }
+ if payload.Kind != "user" {
+ t.Errorf("kind = %q, want user", payload.Kind)
+ }
+}
diff --git a/internal/agent/component/agent.go b/internal/agent/component/agent.go
index 59c3fe425f..e0853f0123 100644
--- a/internal/agent/component/agent.go
+++ b/internal/agent/component/agent.go
@@ -860,6 +860,7 @@ func (c *AgentComponent) invokeNow(ctx context.Context, db *gorm.DB, inputs map[
defer runtime.FinalizeAgentMessage(ctx)
p := mergeAgentParam(c.param, inputs)
+ originalModelID := p.ModelID
hasRuntimeUserPrompt := false
if v, ok := stringFrom(inputs, "user_prompt"); ok {
hasRuntimeUserPrompt = !shouldFallbackToSysQuery(v)
@@ -890,6 +891,11 @@ func (c *AgentComponent) invokeNow(ctx context.Context, db *gorm.DB, inputs map[
}
}
}
+ if state != nil {
+ if err := rejectUnsupportedImages(ctx, db, state, originalModelID, p.ModelID); err != nil {
+ return nil, err
+ }
+ }
if hasRuntimeUserPrompt {
p.UserPrompt = formatAgentRuntimePrompt(inputs, p.UserPrompt)
} else if shouldFallbackToSysQuery(p.UserPrompt) && strings.TrimSpace(p.SystemPrompt) == "" && state != nil {
diff --git a/internal/agent/component/llm.go b/internal/agent/component/llm.go
index 46e39f04c5..adf9eecc1e 100644
--- a/internal/agent/component/llm.go
+++ b/internal/agent/component/llm.go
@@ -735,6 +735,9 @@ func (c *LLMComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[strin
var sysFileImgs []string
hasSysFilesPlaceholder := strings.Contains(p.SystemPrompt, "{sys.files}") || strings.Contains(p.UserPrompt, "{sys.files}")
if state, _, err := runtime.GetStateFromContext[*runtime.CanvasState](ctx); err == nil && state != nil {
+ if err := rejectUnsupportedImages(ctx, db, state, originalModelID, p.ModelID); err != nil {
+ return nil, err
+ }
sysFileTexts, sysFileImgs = collectSysFiles(state)
if len(sysFileImgs) > 0 {
p.VisualFiles = dedupStrings(append(p.VisualFiles, sysFileImgs...))
diff --git a/internal/agent/component/llm_credentials.go b/internal/agent/component/llm_credentials.go
index f675ac6529..c1cc5d3419 100644
--- a/internal/agent/component/llm_credentials.go
+++ b/internal/agent/component/llm_credentials.go
@@ -71,6 +71,42 @@ func resolveChatModelRef(ctx context.Context, db *gorm.DB, modelID, driver, apiK
return modelID, driver, apiKey, baseURL, nil
}
+// modelImageCapability reports an explicitly known image capability. The
+// second result is false when the model is unknown, so local/test drivers keep
+// their historical multimodal behavior.
+func modelImageCapability(ctx context.Context, db *gorm.DB, modelRef, modelName string) (supports, known bool) {
+ if db != nil && modelRef != "" {
+ if row, err := dao.NewTenantModelDAO().GetByID(ctx, db, modelRef); err == nil && row != nil {
+ return entity.ModelType(row.ModelType).Has(entity.ModelTypeImage2Text), true
+ }
+ }
+ if modelName == "" {
+ return false, false
+ }
+ model := dao.GetModelProviderManager().GetModelByNameOrAlias(modelName)
+ if model == nil || len(model.ModelTypes) == 0 {
+ return false, false
+ }
+ for _, typ := range model.ModelTypes {
+ if strings.EqualFold(typ, "vision") || strings.EqualFold(typ, "image2text") {
+ return true, true
+ }
+ }
+ return false, true
+}
+
+func rejectUnsupportedImages(ctx context.Context, db *gorm.DB, state *runtime.CanvasState, modelRef, modelName string) error {
+ if state == nil {
+ return nil
+ }
+ if _, images := collectSysFiles(state); len(images) > 0 {
+ if supports, known := modelImageCapability(ctx, db, modelRef, modelName); known && !supports {
+ return runtime.NewUserFacingError(fmt.Sprintf(`Image input is not supported by the selected model %q. Please select a vision-capable model.`, modelRef))
+ }
+ }
+ return nil
+}
+
// resolveTenantLLMCredentials looks up the old tenant_llm table for the given
// tenant / factory / model. Returns true when credentials were found.
func resolveTenantLLMCredentials(ctx context.Context, db *gorm.DB, tid, driver, modelID, baseURL string) (string, string, bool) {
diff --git a/internal/agent/component/vision_test.go b/internal/agent/component/vision_test.go
index 02d43bcd4c..2b6f116071 100644
--- a/internal/agent/component/vision_test.go
+++ b/internal/agent/component/vision_test.go
@@ -17,7 +17,9 @@
package component
import (
+ "context"
"reflect"
+ "strings"
"testing"
"github.com/cloudwego/eino/schema"
@@ -408,6 +410,26 @@ func TestBuildAgentInputMessagesInjectsSysFiles(t *testing.T) {
}
}
+func TestAgentRejectsImageForKnownTextOnlyModel(t *testing.T) {
+ called := false
+ withAgentRunner(t, func(_ context.Context, _ AgentParam) (*schema.Message, error) {
+ called = true
+ return &schema.Message{Role: schema.Assistant, Content: "unexpected"}, nil
+ })
+ state := runtime.NewCanvasState("run-agent-text-only-image", "task-agent-text-only-image")
+ state.Sys["files"] = []string{"data:image/png;base64,iVBORw0KGgo="}
+
+ _, err := NewAgentComponent(AgentParam{ModelID: "glm-4.7@ZHIPU-AI"}).Invoke(
+ runtime.WithState(t.Context(), state), nil, map[string]any{"user_prompt": "describe this"},
+ )
+ if err == nil || !strings.Contains(err.Error(), "Image input is not supported by the selected model") {
+ t.Fatalf("Invoke error = %v, want clear unsupported-image error", err)
+ }
+ if called {
+ t.Fatal("text-only model must be rejected before agentRunner")
+ }
+}
+
// TestBuildAgentInputMessagesSysFilesTextOnly: non-image uploads merge into
// the plain string Content without creating multi-modal parts.
func TestBuildAgentInputMessagesSysFilesTextOnly(t *testing.T) {
diff --git a/internal/agent/runtime/component.go b/internal/agent/runtime/component.go
index a6bedb4a71..35dfa40b58 100644
--- a/internal/agent/runtime/component.go
+++ b/internal/agent/runtime/component.go
@@ -64,6 +64,17 @@ type ComponentFactory func(name string, params map[string]any) (Component, error
// the message is for humans, the sentinel is for code.
var ErrNotImplemented = fmt.Errorf("component: not yet implemented (placeholder)")
+// UserFacingError marks a validated request failure whose message is safe to
+// return directly to the Agent client without canvas execution prefixes.
+type UserFacingError struct{ Err error }
+
+func (e *UserFacingError) Error() string { return e.Err.Error() }
+func (e *UserFacingError) Unwrap() error { return e.Err }
+
+func NewUserFacingError(message string) error {
+ return &UserFacingError{Err: fmt.Errorf("%s", message)}
+}
+
// ParamError wraps a parameter validation failure with the field name
// for clearer error messages to the user.
type ParamError struct {
diff --git a/internal/entity/models/zhipu_ai_test.go b/internal/entity/models/zhipu_ai_test.go
index 6895acd923..09ebadfdd0 100644
--- a/internal/entity/models/zhipu_ai_test.go
+++ b/internal/entity/models/zhipu_ai_test.go
@@ -17,6 +17,41 @@ func newZhipuAIForTest(baseURL string) *ZhipuAIModel {
)
}
+func TestZhipuAIChatForwardsMultimodalContentBlocks(t *testing.T) {
+ withSSRFBypass(t)
+ var request map[string]any
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/chat/completions" {
+ t.Errorf("path=%s, want /chat/completions", r.URL.Path)
+ }
+ if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
+ t.Errorf("decode request: %v", err)
+ }
+ _ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": "ok"}}}})
+ }))
+ defer srv.Close()
+
+ key := "test-key"
+ _, err := NewZhipuAIModel(map[string]string{"default": srv.URL}, URLSuffix{Chat: "chat/completions"}).ChatWithMessages(
+ t.Context(), "glm-4.7", []Message{{Role: "user", Content: []interface{}{
+ map[string]any{"type": "text", "text": "describe"},
+ map[string]any{"type": "image_url", "image_url": map[string]any{"url": "data:image/png;base64,AA=="}},
+ }}}, &APIConfig{ApiKey: &key}, nil, nil,
+ )
+ if err != nil {
+ t.Fatalf("ChatWithMessages: %v", err)
+ }
+ messages, ok := request["messages"].([]any)
+ if !ok || len(messages) != 1 {
+ t.Fatalf("messages = %#v, want one request message", request["messages"])
+ }
+ message := messages[0].(map[string]any)
+ parts, ok := message["content"].([]any)
+ if !ok || len(parts) != 2 || parts[1].(map[string]any)["type"] != "image_url" {
+ t.Fatalf("content = %#v, want text + image_url blocks", message["content"])
+ }
+}
+
func writeZhipuAITestAudio(t *testing.T) string {
t.Helper()