Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(agent): describe uploads when the chat model cannot see images (#4219)

AsanoReo75c1178008ce
diff --git a/internal/agent/engine.go b/internal/agent/engine.go
index 8694477fa..7fc477025 100644
--- a/internal/agent/engine.go
+++ b/internal/agent/engine.go
@@ -1493,6 +1493,18 @@ const (
toolImageDescribeTimeout = 120 * time.Second
)
+// DescribeDataURIImages turns user-uploaded data URIs into text with the
+// configured VLM. A chat model that cannot see images uses this instead of
+// dropping the upload. Uploads ask for a full read; tool results keep the
+// prompt that came with the tool call.
+func (e *AgentEngine) DescribeDataURIImages(ctx context.Context, imageDataURIs []string) []string {
+ prompts := make([]string, len(imageDataURIs))
+ for i := range prompts {
+ prompts[i] = toolImageFullPrompt
+ }
+ return e.describeImages(ctx, imageDataURIs, prompts)
+}
+
// describeImages asks the configured VLM to describe tool result images, a
// few at a time, each under its own timeout so a slow image cannot starve the
// rest. prompts is aligned with the images (nil means a brief description for
diff --git a/internal/agent/engine_image_test.go b/internal/agent/engine_image_test.go
index 03db91f93..7af49908f 100644
--- a/internal/agent/engine_image_test.go
+++ b/internal/agent/engine_image_test.go
@@ -70,8 +70,10 @@ func TestDecodeDataURIBytes_EmptyBase64(t *testing.T) {
}
func TestDescribeImages_WithDescriber(t *testing.T) {
+ var prompt string
engine := &AgentEngine{
- imageDescriber: func(ctx context.Context, imgBytes []byte, prompt string) (string, error) {
+ imageDescriber: func(_ context.Context, _ []byte, got string) (string, error) {
+ prompt = got
return "A red square image", nil
},
}
@@ -86,6 +88,14 @@ func TestDescribeImages_WithDescriber(t *testing.T) {
if descriptions[0] != "A red square image" {
t.Errorf("unexpected description: %s", descriptions[0])
}
+
+ exported := engine.DescribeDataURIImages(context.Background(), []string{dataURI})
+ if len(exported) != 1 || exported[0] != "A red square image" {
+ t.Fatalf("exported description = %v", exported)
+ }
+ if prompt != toolImageFullPrompt {
+ t.Fatalf("upload prompt = %q", prompt)
+ }
}
func TestDescribeImages_VLMFailure(t *testing.T) {
diff --git a/internal/application/service/session_agent_image_test.go b/internal/application/service/session_agent_image_test.go
new file mode 100644
index 000000000..37064056b
--- /dev/null
+++ b/internal/application/service/session_agent_image_test.go
@@ -0,0 +1,55 @@
+package service
+
+import (
+ "context"
+ "strings"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/agent"
+)
+
+// The QA path type-asserts the engine. This fails the build if that method moves.
+var _ dataURIImageDescriber = (*agent.AgentEngine)(nil)
+
+func TestComposeAgentImageQueryPassesImagesToAVisionModel(t *testing.T) {
+ query, images := composeAgentImageQuery(context.Background(), "看这张图", "", true,
+ []string{"data:image/png;base64,AAAA"},
+ func(context.Context, []string) []string {
+ t.Fatal("vision models should receive the image, not a description")
+ return nil
+ })
+ if query != "看这张图" || len(images) != 1 {
+ t.Fatalf("query=%q images=%v", query, images)
+ }
+}
+
+func TestComposeAgentImageQueryUsesVLMWhenTheChatModelCannotSee(t *testing.T) {
+ var got []string
+ query, images := composeAgentImageQuery(context.Background(), "看这张图", "", false,
+ []string{"data:image/png;base64,AAAA"},
+ func(_ context.Context, urls []string) []string {
+ got = urls
+ return []string{" 一张收据 ", ""}
+ })
+ if len(images) != 0 {
+ t.Fatalf("text model still received images: %v", images)
+ }
+ if !strings.Contains(query, "一张收据") || !strings.Contains(query, "[用户上传图片内容]") {
+ t.Fatalf("query = %q", query)
+ }
+ if len(got) != 1 {
+ t.Fatalf("describer urls = %v", got)
+ }
+}
+
+func TestComposeAgentImageQueryKeepsAnExistingDescription(t *testing.T) {
+ query, images := composeAgentImageQuery(context.Background(), "看这张图", "已有说明", false,
+ []string{"data:image/png;base64,AAAA"},
+ func(context.Context, []string) []string {
+ t.Fatal("an existing description should not call the VLM again")
+ return nil
+ })
+ if len(images) != 0 || !strings.Contains(query, "已有说明") {
+ t.Fatalf("query=%q images=%v", query, images)
+ }
+}
diff --git a/internal/application/service/session_agent_qa.go b/internal/application/service/session_agent_qa.go
index 3157b8c41..766055b4a 100644
--- a/internal/application/service/session_agent_qa.go
+++ b/internal/application/service/session_agent_qa.go
@@ -7,6 +7,7 @@ import (
"fmt"
"slices"
"strings"
+ "time"
"github.com/Tencent/WeKnora/internal/agent"
"github.com/Tencent/WeKnora/internal/agent/collaboration"
@@ -378,27 +379,33 @@ func (s *sessionService) AgentQA(
engine.SetContextCheckpointSink(messageCheckpointSink{repo: s.messageRepo, sessionID: sessionID})
}
- agentQuery := req.Query
+ var agentQuery string
var agentImageURLs []string
- if agentModelSupportsVision && len(req.ImageURLs) > 0 {
- agentImageURLs = req.ImageURLs
- logger.Infof(ctx, "Agent model supports vision, passing %d image(s) directly", len(agentImageURLs))
- } else if req.ImageDescription != "" {
- agentQuery = req.Query + "\n\n[用户上传图片内容]\n" + req.ImageDescription
- logger.Infof(ctx, "Agent model does not support vision, appending image description (%d chars)", len(req.ImageDescription))
- }
- if req.QuotedContext != "" {
- agentQuery += "\n\n" + req.QuotedContext
- }
- // Stored documents are listed for read_file rather than pasted in;
- // inline uploads without a stored document still carry their content.
- if len(req.Attachments) > 0 {
- agentQuery += req.Attachments.BuildAgentPrompt()
- logger.Infof(ctx, "Appended %d attachment(s) to agent query", len(req.Attachments))
- }
- if manifest := buildSandboxAttachmentsPrompt(stagedAttachments, layout); manifest != "" {
- agentQuery += manifest
- logger.Infof(ctx, "Appended %d staged sandbox attachment path(s) to agent query", len(stagedAttachments))
+ if req.Resume == nil {
+ agentQuery, agentImageURLs = composeAgentImageQuery(
+ ctx, req.Query, req.ImageDescription, agentModelSupportsVision, req.ImageURLs,
+ agentImageDescriber(engine),
+ )
+ if agentModelSupportsVision && len(agentImageURLs) > 0 {
+ logger.Infof(ctx, "Agent model supports vision, passing %d image(s) directly", len(agentImageURLs))
+ } else if len(req.ImageURLs) > 0 && strings.Contains(agentQuery, "[用户上传图片内容]") {
+ logger.Infof(ctx, "Agent model does not support vision, appending image description")
+ } else if len(req.ImageURLs) > 0 {
+ logger.Warnf(ctx, "Agent model cannot view %d image(s) and no description was produced", len(req.ImageURLs))
+ }
+ if req.QuotedContext != "" {
+ agentQuery += "\n\n" + req.QuotedContext
+ }
+ // Stored documents are listed for read_file rather than pasted in;
+ // inline uploads without a stored document still carry their content.
+ if len(req.Attachments) > 0 {
+ agentQuery += req.Attachments.BuildAgentPrompt()
+ logger.Infof(ctx, "Appended %d attachment(s) to agent query", len(req.Attachments))
+ }
+ if manifest := buildSandboxAttachmentsPrompt(stagedAttachments, layout); manifest != "" {
+ agentQuery += manifest
+ logger.Infof(ctx, "Appended %d staged sandbox attachment path(s) to agent query", len(stagedAttachments))
+ }
}
// Scope envelopes (runtime_context / must_use) are injected per LLM call inside
@@ -431,6 +438,51 @@ func (s *sessionService) AgentQA(
return nil
}
+type dataURIImageDescriber interface {
+ DescribeDataURIImages(context.Context, []string) []string
+}
+
+func agentImageDescriber(engine any) func(context.Context, []string) []string {
+ describer, ok := engine.(dataURIImageDescriber)
+ if !ok || describer == nil {
+ return nil
+ }
+ return func(ctx context.Context, urls []string) []string {
+ imageCtx, cancel := context.WithTimeout(ctx, 60*time.Second)
+ defer cancel()
+ return describer.DescribeDataURIImages(imageCtx, urls)
+ }
+}
+
+// composeAgentImageQuery keeps images on the query when the chat model can
+// see them. Otherwise it uses an existing description, or asks the VLM
+// describer, so a text-only agent still receives the upload.
+func composeAgentImageQuery(
+ ctx context.Context,
+ query, description string,
+ vision bool,
+ urls []string,
+ describe func(context.Context, []string) []string,
+) (string, []string) {
+ if vision && len(urls) > 0 {
+ return query, urls
+ }
+ text := strings.TrimSpace(description)
+ if text == "" && len(urls) > 0 && describe != nil {
+ var kept []string
+ for _, part := range describe(ctx, urls) {
+ if part = strings.TrimSpace(part); part != "" {
+ kept = append(kept, part)
+ }
+ }
+ text = strings.Join(kept, "\n")
+ }
+ if text != "" {
+ return query + "\n\n[用户上传图片内容]\n" + text, nil
+ }
+ return query, nil
+}
+
// resumableAgentRequest keeps the durable part of req needed to rebuild the
// run elsewhere. Session, attachments and images are reloaded from storage.
func resumableAgentRequest(ctx context.Context, req *types.QARequest) types.ResumableAgentRequest {