Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(agent): describe uploads when the chat model cannot see images (#4219)
AsanoReo75c1178008ce
diff --git a/internal/agent/engine.go b/internal/agent/engine.goindex 8694477fa..7fc477025 100644--- a/internal/agent/engine.go+++ b/internal/agent/engine.go@@ -1493,6 +1493,18 @@ const (toolImageDescribeTimeout = 120 * time.Second)+// DescribeDataURIImages turns user-uploaded data URIs into text with the+// configured VLM. A chat model that cannot see images uses this instead of+// dropping the upload. Uploads ask for a full read; tool results keep the+// prompt that came with the tool call.+func (e *AgentEngine) DescribeDataURIImages(ctx context.Context, imageDataURIs []string) []string {+ prompts := make([]string, len(imageDataURIs))+ for i := range prompts {+ prompts[i] = toolImageFullPrompt+ }+ return e.describeImages(ctx, imageDataURIs, prompts)+}+// describeImages asks the configured VLM to describe tool result images, a// few at a time, each under its own timeout so a slow image cannot starve the// rest. prompts is aligned with the images (nil means a brief description fordiff --git a/internal/agent/engine_image_test.go b/internal/agent/engine_image_test.goindex 03db91f93..7af49908f 100644--- a/internal/agent/engine_image_test.go+++ b/internal/agent/engine_image_test.go@@ -70,8 +70,10 @@ func TestDecodeDataURIBytes_EmptyBase64(t *testing.T) {}func TestDescribeImages_WithDescriber(t *testing.T) {+ var prompt stringengine := &AgentEngine{- imageDescriber: func(ctx context.Context, imgBytes []byte, prompt string) (string, error) {+ imageDescriber: func(_ context.Context, _ []byte, got string) (string, error) {+ prompt = gotreturn "A red square image", nil},}@@ -86,6 +88,14 @@ func TestDescribeImages_WithDescriber(t *testing.T) {if descriptions[0] != "A red square image" {t.Errorf("unexpected description: %s", descriptions[0])}++ exported := engine.DescribeDataURIImages(context.Background(), []string{dataURI})+ if len(exported) != 1 || exported[0] != "A red square image" {+ t.Fatalf("exported description = %v", exported)+ }+ if prompt != toolImageFullPrompt {+ t.Fatalf("upload prompt = %q", prompt)+ }}func TestDescribeImages_VLMFailure(t *testing.T) {diff --git a/internal/application/service/session_agent_image_test.go b/internal/application/service/session_agent_image_test.gonew file mode 100644index 000000000..37064056b--- /dev/null+++ b/internal/application/service/session_agent_image_test.go@@ -0,0 +1,55 @@+package service++import (+ "context"+ "strings"+ "testing"++ "github.com/Tencent/WeKnora/internal/agent"+)++// The QA path type-asserts the engine. This fails the build if that method moves.+var _ dataURIImageDescriber = (*agent.AgentEngine)(nil)++func TestComposeAgentImageQueryPassesImagesToAVisionModel(t *testing.T) {+ query, images := composeAgentImageQuery(context.Background(), "看这张图", "", true,+ []string{"data:image/png;base64,AAAA"},+ func(context.Context, []string) []string {+ t.Fatal("vision models should receive the image, not a description")+ return nil+ })+ if query != "看这张图" || len(images) != 1 {+ t.Fatalf("query=%q images=%v", query, images)+ }+}++func TestComposeAgentImageQueryUsesVLMWhenTheChatModelCannotSee(t *testing.T) {+ var got []string+ query, images := composeAgentImageQuery(context.Background(), "看这张图", "", false,+ []string{"data:image/png;base64,AAAA"},+ func(_ context.Context, urls []string) []string {+ got = urls+ return []string{" 一张收据 ", ""}+ })+ if len(images) != 0 {+ t.Fatalf("text model still received images: %v", images)+ }+ if !strings.Contains(query, "一张收据") || !strings.Contains(query, "[用户上传图片内容]") {+ t.Fatalf("query = %q", query)+ }+ if len(got) != 1 {+ t.Fatalf("describer urls = %v", got)+ }+}++func TestComposeAgentImageQueryKeepsAnExistingDescription(t *testing.T) {+ query, images := composeAgentImageQuery(context.Background(), "看这张图", "已有说明", false,+ []string{"data:image/png;base64,AAAA"},+ func(context.Context, []string) []string {+ t.Fatal("an existing description should not call the VLM again")+ return nil+ })+ if len(images) != 0 || !strings.Contains(query, "已有说明") {+ t.Fatalf("query=%q images=%v", query, images)+ }+}diff --git a/internal/application/service/session_agent_qa.go b/internal/application/service/session_agent_qa.goindex 3157b8c41..766055b4a 100644--- a/internal/application/service/session_agent_qa.go+++ b/internal/application/service/session_agent_qa.go@@ -7,6 +7,7 @@ import ("fmt""slices""strings"+ "time""github.com/Tencent/WeKnora/internal/agent""github.com/Tencent/WeKnora/internal/agent/collaboration"@@ -378,27 +379,33 @@ func (s *sessionService) AgentQA(engine.SetContextCheckpointSink(messageCheckpointSink{repo: s.messageRepo, sessionID: sessionID})}- agentQuery := req.Query+ var agentQuery stringvar agentImageURLs []string- if agentModelSupportsVision && len(req.ImageURLs) > 0 {- agentImageURLs = req.ImageURLs- logger.Infof(ctx, "Agent model supports vision, passing %d image(s) directly", len(agentImageURLs))- } else if req.ImageDescription != "" {- agentQuery = req.Query + "\n\n[用户上传图片内容]\n" + req.ImageDescription- logger.Infof(ctx, "Agent model does not support vision, appending image description (%d chars)", len(req.ImageDescription))- }- if req.QuotedContext != "" {- agentQuery += "\n\n" + req.QuotedContext- }- // Stored documents are listed for read_file rather than pasted in;- // inline uploads without a stored document still carry their content.- if len(req.Attachments) > 0 {- agentQuery += req.Attachments.BuildAgentPrompt()- logger.Infof(ctx, "Appended %d attachment(s) to agent query", len(req.Attachments))- }- if manifest := buildSandboxAttachmentsPrompt(stagedAttachments, layout); manifest != "" {- agentQuery += manifest- logger.Infof(ctx, "Appended %d staged sandbox attachment path(s) to agent query", len(stagedAttachments))+ if req.Resume == nil {+ agentQuery, agentImageURLs = composeAgentImageQuery(+ ctx, req.Query, req.ImageDescription, agentModelSupportsVision, req.ImageURLs,+ agentImageDescriber(engine),+ )+ if agentModelSupportsVision && len(agentImageURLs) > 0 {+ logger.Infof(ctx, "Agent model supports vision, passing %d image(s) directly", len(agentImageURLs))+ } else if len(req.ImageURLs) > 0 && strings.Contains(agentQuery, "[用户上传图片内容]") {+ logger.Infof(ctx, "Agent model does not support vision, appending image description")+ } else if len(req.ImageURLs) > 0 {+ logger.Warnf(ctx, "Agent model cannot view %d image(s) and no description was produced", len(req.ImageURLs))+ }+ if req.QuotedContext != "" {+ agentQuery += "\n\n" + req.QuotedContext+ }+ // Stored documents are listed for read_file rather than pasted in;+ // inline uploads without a stored document still carry their content.+ if len(req.Attachments) > 0 {+ agentQuery += req.Attachments.BuildAgentPrompt()+ logger.Infof(ctx, "Appended %d attachment(s) to agent query", len(req.Attachments))+ }+ if manifest := buildSandboxAttachmentsPrompt(stagedAttachments, layout); manifest != "" {+ agentQuery += manifest+ logger.Infof(ctx, "Appended %d staged sandbox attachment path(s) to agent query", len(stagedAttachments))+ }}// Scope envelopes (runtime_context / must_use) are injected per LLM call inside@@ -431,6 +438,51 @@ func (s *sessionService) AgentQA(return nil}+type dataURIImageDescriber interface {+ DescribeDataURIImages(context.Context, []string) []string+}++func agentImageDescriber(engine any) func(context.Context, []string) []string {+ describer, ok := engine.(dataURIImageDescriber)+ if !ok || describer == nil {+ return nil+ }+ return func(ctx context.Context, urls []string) []string {+ imageCtx, cancel := context.WithTimeout(ctx, 60*time.Second)+ defer cancel()+ return describer.DescribeDataURIImages(imageCtx, urls)+ }+}++// composeAgentImageQuery keeps images on the query when the chat model can+// see them. Otherwise it uses an existing description, or asks the VLM+// describer, so a text-only agent still receives the upload.+func composeAgentImageQuery(+ ctx context.Context,+ query, description string,+ vision bool,+ urls []string,+ describe func(context.Context, []string) []string,+) (string, []string) {+ if vision && len(urls) > 0 {+ return query, urls+ }+ text := strings.TrimSpace(description)+ if text == "" && len(urls) > 0 && describe != nil {+ var kept []string+ for _, part := range describe(ctx, urls) {+ if part = strings.TrimSpace(part); part != "" {+ kept = append(kept, part)+ }+ }+ text = strings.Join(kept, "\n")+ }+ if text != "" {+ return query + "\n\n[用户上传图片内容]\n" + text, nil+ }+ return query, nil+}+// resumableAgentRequest keeps the durable part of req needed to rebuild the// run elsewhere. Session, attachments and images are reloaded from storage.func resumableAgentRequest(ctx context.Context, req *types.QARequest) types.ResumableAgentRequest {