RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(agent): reject unsupported image input cleanly (#19539)
Sbaaoui Idriss815639482c2a
diff --git a/internal/agent/canvas/run_error.go b/internal/agent/canvas/run_error.goindex 3f1b3367e6..8fa2e365a4 100644--- a/internal/agent/canvas/run_error.go+++ b/internal/agent/canvas/run_error.go@@ -14,7 +14,11 @@package canvas-import "errors"+import (+ "errors"++ "ragflow/internal/agent/runtime"+)const (// RunErrorKindInternal marks an error whose cause is safe for server logs@@ -48,6 +52,10 @@ func (e *internalRunError) Unwrap() error {}func runErrorEvent(err error) ErrorEvent {+ var userErr *runtime.UserFacingError+ if errors.As(err, &userErr) {+ return ErrorEvent{Message: userErr.Error(), Kind: "user"}+ }var internalErr *internalRunErrorif errors.As(err, &internalErr) {return ErrorEvent{diff --git a/internal/agent/canvas/run_error_test.go b/internal/agent/canvas/run_error_test.goindex 61ec997c73..8e9a76f2d4 100644--- a/internal/agent/canvas/run_error_test.go+++ b/internal/agent/canvas/run_error_test.go@@ -21,6 +21,8 @@ import ("fmt""strings""testing"++ "ragflow/internal/agent/runtime")func TestRunnerRedactsInternalRunError(t *testing.T) {@@ -69,3 +71,13 @@ func TestRunErrorEventPreservesPublicMessage(t *testing.T) {t.Errorf("kind = %q, want empty for public runtime error", payload.Kind)}}++func TestRunErrorEventPreservesUserFacingMessage(t *testing.T) {+ payload := runErrorEvent(runtime.NewUserFacingError("Image input is not supported by the selected model \"glm-4.7\". Please select a vision-capable model."))+ if payload.Message != "Image input is not supported by the selected model \"glm-4.7\". Please select a vision-capable model." {+ t.Errorf("message = %q, want clean user-facing message", payload.Message)+ }+ if payload.Kind != "user" {+ t.Errorf("kind = %q, want user", payload.Kind)+ }+}diff --git a/internal/agent/component/agent.go b/internal/agent/component/agent.goindex 59c3fe425f..e0853f0123 100644--- a/internal/agent/component/agent.go+++ b/internal/agent/component/agent.go@@ -860,6 +860,7 @@ func (c *AgentComponent) invokeNow(ctx context.Context, db *gorm.DB, inputs map[defer runtime.FinalizeAgentMessage(ctx)p := mergeAgentParam(c.param, inputs)+ originalModelID := p.ModelIDhasRuntimeUserPrompt := falseif v, ok := stringFrom(inputs, "user_prompt"); ok {hasRuntimeUserPrompt = !shouldFallbackToSysQuery(v)@@ -890,6 +891,11 @@ func (c *AgentComponent) invokeNow(ctx context.Context, db *gorm.DB, inputs map[}}}+ if state != nil {+ if err := rejectUnsupportedImages(ctx, db, state, originalModelID, p.ModelID); err != nil {+ return nil, err+ }+ }if hasRuntimeUserPrompt {p.UserPrompt = formatAgentRuntimePrompt(inputs, p.UserPrompt)} else if shouldFallbackToSysQuery(p.UserPrompt) && strings.TrimSpace(p.SystemPrompt) == "" && state != nil {diff --git a/internal/agent/component/llm.go b/internal/agent/component/llm.goindex 46e39f04c5..adf9eecc1e 100644--- a/internal/agent/component/llm.go+++ b/internal/agent/component/llm.go@@ -735,6 +735,9 @@ func (c *LLMComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[strinvar sysFileImgs []stringhasSysFilesPlaceholder := strings.Contains(p.SystemPrompt, "{sys.files}") || strings.Contains(p.UserPrompt, "{sys.files}")if state, _, err := runtime.GetStateFromContext[*runtime.CanvasState](ctx); err == nil && state != nil {+ if err := rejectUnsupportedImages(ctx, db, state, originalModelID, p.ModelID); err != nil {+ return nil, err+ }sysFileTexts, sysFileImgs = collectSysFiles(state)if len(sysFileImgs) > 0 {p.VisualFiles = dedupStrings(append(p.VisualFiles, sysFileImgs...))diff --git a/internal/agent/component/llm_credentials.go b/internal/agent/component/llm_credentials.goindex f675ac6529..c1cc5d3419 100644--- a/internal/agent/component/llm_credentials.go+++ b/internal/agent/component/llm_credentials.go@@ -71,6 +71,42 @@ func resolveChatModelRef(ctx context.Context, db *gorm.DB, modelID, driver, apiKreturn modelID, driver, apiKey, baseURL, nil}+// modelImageCapability reports an explicitly known image capability. The+// second result is false when the model is unknown, so local/test drivers keep+// their historical multimodal behavior.+func modelImageCapability(ctx context.Context, db *gorm.DB, modelRef, modelName string) (supports, known bool) {+ if db != nil && modelRef != "" {+ if row, err := dao.NewTenantModelDAO().GetByID(ctx, db, modelRef); err == nil && row != nil {+ return entity.ModelType(row.ModelType).Has(entity.ModelTypeImage2Text), true+ }+ }+ if modelName == "" {+ return false, false+ }+ model := dao.GetModelProviderManager().GetModelByNameOrAlias(modelName)+ if model == nil || len(model.ModelTypes) == 0 {+ return false, false+ }+ for _, typ := range model.ModelTypes {+ if strings.EqualFold(typ, "vision") || strings.EqualFold(typ, "image2text") {+ return true, true+ }+ }+ return false, true+}++func rejectUnsupportedImages(ctx context.Context, db *gorm.DB, state *runtime.CanvasState, modelRef, modelName string) error {+ if state == nil {+ return nil+ }+ if _, images := collectSysFiles(state); len(images) > 0 {+ if supports, known := modelImageCapability(ctx, db, modelRef, modelName); known && !supports {+ return runtime.NewUserFacingError(fmt.Sprintf(`Image input is not supported by the selected model %q. Please select a vision-capable model.`, modelRef))+ }+ }+ return nil+}+// resolveTenantLLMCredentials looks up the old tenant_llm table for the given// tenant / factory / model. Returns true when credentials were found.func resolveTenantLLMCredentials(ctx context.Context, db *gorm.DB, tid, driver, modelID, baseURL string) (string, string, bool) {diff --git a/internal/agent/component/vision_test.go b/internal/agent/component/vision_test.goindex 02d43bcd4c..2b6f116071 100644--- a/internal/agent/component/vision_test.go+++ b/internal/agent/component/vision_test.go@@ -17,7 +17,9 @@package componentimport (+ "context""reflect"+ "strings""testing""github.com/cloudwego/eino/schema"@@ -408,6 +410,26 @@ func TestBuildAgentInputMessagesInjectsSysFiles(t *testing.T) {}}+func TestAgentRejectsImageForKnownTextOnlyModel(t *testing.T) {+ called := false+ withAgentRunner(t, func(_ context.Context, _ AgentParam) (*schema.Message, error) {+ called = true+ return &schema.Message{Role: schema.Assistant, Content: "unexpected"}, nil+ })+ state := runtime.NewCanvasState("run-agent-text-only-image", "task-agent-text-only-image")+ state.Sys["files"] = []string{"data:image/png;base64,iVBORw0KGgo="}++ _, err := NewAgentComponent(AgentParam{ModelID: "glm-4.7@ZHIPU-AI"}).Invoke(+ runtime.WithState(t.Context(), state), nil, map[string]any{"user_prompt": "describe this"},+ )+ if err == nil || !strings.Contains(err.Error(), "Image input is not supported by the selected model") {+ t.Fatalf("Invoke error = %v, want clear unsupported-image error", err)+ }+ if called {+ t.Fatal("text-only model must be rejected before agentRunner")+ }+}+// TestBuildAgentInputMessagesSysFilesTextOnly: non-image uploads merge into// the plain string Content without creating multi-modal parts.func TestBuildAgentInputMessagesSysFilesTextOnly(t *testing.T) {diff --git a/internal/agent/runtime/component.go b/internal/agent/runtime/component.goindex a6bedb4a71..35dfa40b58 100644--- a/internal/agent/runtime/component.go+++ b/internal/agent/runtime/component.go@@ -64,6 +64,17 @@ type ComponentFactory func(name string, params map[string]any) (Component, error// the message is for humans, the sentinel is for code.var ErrNotImplemented = fmt.Errorf("component: not yet implemented (placeholder)")+// UserFacingError marks a validated request failure whose message is safe to+// return directly to the Agent client without canvas execution prefixes.+type UserFacingError struct{ Err error }++func (e *UserFacingError) Error() string { return e.Err.Error() }+func (e *UserFacingError) Unwrap() error { return e.Err }++func NewUserFacingError(message string) error {+ return &UserFacingError{Err: fmt.Errorf("%s", message)}+}+// ParamError wraps a parameter validation failure with the field name// for clearer error messages to the user.type ParamError struct {diff --git a/internal/entity/models/zhipu_ai_test.go b/internal/entity/models/zhipu_ai_test.goindex 6895acd923..09ebadfdd0 100644--- a/internal/entity/models/zhipu_ai_test.go+++ b/internal/entity/models/zhipu_ai_test.go@@ -17,6 +17,41 @@ func newZhipuAIForTest(baseURL string) *ZhipuAIModel {)}+func TestZhipuAIChatForwardsMultimodalContentBlocks(t *testing.T) {+ withSSRFBypass(t)+ var request map[string]any+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/chat/completions" {+ t.Errorf("path=%s, want /chat/completions", r.URL.Path)+ }+ if err := json.NewDecoder(r.Body).Decode(&request); err != nil {+ t.Errorf("decode request: %v", err)+ }+ _ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": "ok"}}}})+ }))+ defer srv.Close()++ key := "test-key"+ _, err := NewZhipuAIModel(map[string]string{"default": srv.URL}, URLSuffix{Chat: "chat/completions"}).ChatWithMessages(+ t.Context(), "glm-4.7", []Message{{Role: "user", Content: []interface{}{+ map[string]any{"type": "text", "text": "describe"},+ map[string]any{"type": "image_url", "image_url": map[string]any{"url": "data:image/png;base64,AA=="}},+ }}}, &APIConfig{ApiKey: &key}, nil, nil,+ )+ if err != nil {+ t.Fatalf("ChatWithMessages: %v", err)+ }+ messages, ok := request["messages"].([]any)+ if !ok || len(messages) != 1 {+ t.Fatalf("messages = %#v, want one request message", request["messages"])+ }+ message := messages[0].(map[string]any)+ parts, ok := message["content"].([]any)+ if !ok || len(parts) != 2 || parts[1].(map[string]any)["type"] != "image_url" {+ t.Fatalf("content = %#v, want text + image_url blocks", message["content"])+ }+}+func writeZhipuAITestAudio(t *testing.T) string {t.Helper()