Skip to content

An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作

:bug: Avoid empty state text in multimodal decisions https://github.com/siyuan-note/siyuan/issues/19986

Daniel88869b9f8052
diff --git a/kernel/agent/decision_test.go b/kernel/agent/decision_test.go
index a77fe17839..db74f32bf2 100644
--- a/kernel/agent/decision_test.go
+++ b/kernel/agent/decision_test.go
@@ -73,7 +73,7 @@ func TestDecisionNativeJSONPrecision(t *testing.T) {
t.Error(err)
return
}
- if len(payload.Input) != 1 || len(payload.Input[0].Content) != 2 || string(payload.Input[0].Content[1].Data) != evidence {
+ if len(payload.Input) != 1 || len(payload.Input[0].Content) != 1 || string(payload.Input[0].Content[0].Data) != evidence {
t.Errorf("native JSON evidence changed: %+v", payload)
}
io.WriteString(w, `{"model":"gpt-6-luna","answers":[{"name":"q","type":"predicate","probability":0.8}],"usage":{"input_tokens":20,"output_tokens":0}}`)
diff --git a/kernel/mcp/tools/decision_image_test.go b/kernel/mcp/tools/decision_image_test.go
index b62b111d52..54fb63885a 100644
--- a/kernel/mcp/tools/decision_image_test.go
+++ b/kernel/mcp/tools/decision_image_test.go
@@ -108,7 +108,7 @@ func TestDecisionOpenAIToolJSONAndQuestionImages(t *testing.T) {
t.Error(err)
return
}
- if len(payload.Input) != 1 || len(payload.Input[0].Content) != 3 || string(payload.Input[0].Content[2].Data) != `{"priority":2}` || !strings.HasPrefix(payload.Input[0].Content[1].ImageURL, "data:image/png;base64,") || !strings.Contains(string(payload.Questions[0].Instructions), "https://example.com/candidate.png") {
+ if len(payload.Input) != 1 || len(payload.Input[0].Content) != 2 || string(payload.Input[0].Content[1].Data) != `{"priority":2}` || !strings.HasPrefix(payload.Input[0].Content[0].ImageURL, "data:image/png;base64,") || !strings.Contains(string(payload.Questions[0].Instructions), "https://example.com/candidate.png") {
t.Errorf("native tool input lost: %+v", payload)
}
io.WriteString(w, `{"model":"gpt-6-luna","answers":[{"name":"q","type":"predicate","probability":0.9}],"usage":{"input_tokens":100,"output_tokens":0,"input_tokens_details":{"cached_tokens":40,"cache_write_tokens":60}}}`)
diff --git a/kernel/util/decision_input.go b/kernel/util/decision_input.go
index b638974ac9..5de210c0e0 100644
--- a/kernel/util/decision_input.go
+++ b/kernel/util/decision_input.go
@@ -147,6 +147,11 @@ func openAIDecisionInput(state DecisionState) (json.RawMessage, error) {
if len(data) == 0 && len(state.Images) == 0 {
return json.Marshal(string(text))
}
+ // JSON 或图片单独作为证据时,不附加表示空状态的 "{}" 文本。
+ // 这不是用户输入,可能干扰图片判断;显式提供的文本或 JSON 仍完整保留。
+ if state.Context == "" && state.Text == "" && len(state.Blocks) == 0 {
+ text = nil
+ }
parts, err := openAIDecisionContent(string(text), state.Images)
if err != nil {
return nil, err
diff --git a/kernel/util/decision_input_test.go b/kernel/util/decision_input_test.go
index 5fbf52255c..acfd80116f 100644
--- a/kernel/util/decision_input_test.go
+++ b/kernel/util/decision_input_test.go
@@ -144,3 +144,45 @@ func TestDecisionImageValidation(t *testing.T) {
}
}
}
+
+func TestDecisionOpenAIMultimodalInputOmitsEmptyState(t *testing.T) {
+ image := DecisionImage{URL: "https://example.com/explicit.png"}
+ for _, tc := range []struct {
+ name string
+ state DecisionState
+ types []string
+ }{
+ {"image only", DecisionState{Images: []DecisionImage{image}}, []string{"input_image"}},
+ {"JSON only", DecisionState{Data: json.RawMessage(`{}`)}, []string{"input_json"}},
+ {"JSON and image", DecisionState{Data: json.RawMessage(`{"id":9007199254740993}`), Images: []DecisionImage{image}}, []string{"input_image", "input_json"}},
+ {"explicit text", DecisionState{Text: "{}", Images: []DecisionImage{image}}, []string{"input_text", "input_image"}},
+ {"shared context", DecisionState{Context: "Reference image", Images: []DecisionImage{image}}, []string{"input_text", "input_image"}},
+ {"blocks", DecisionState{Blocks: []DecisionBlock{{ID: "block", Markdown: "Evidence"}}, Images: []DecisionImage{image}}, []string{"input_text", "input_image"}},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ input, err := openAIDecisionInput(tc.state)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var messages []struct {
+ Content []openAIDecisionPart `json:"content"`
+ }
+ if err := json.Unmarshal(input, &messages); err != nil || len(messages) != 1 {
+ t.Fatalf("invalid input envelope: %s: %v", input, err)
+ }
+ var types []string
+ for _, part := range messages[0].Content {
+ types = append(types, part.Type)
+ if part.Type == "input_text" && part.Text == "{}" {
+ t.Fatal("synthetic empty-state text was sent as evidence")
+ }
+ if part.Type == "input_json" && string(part.Data) != string(tc.state.Data) {
+ t.Fatal("explicit JSON evidence changed")
+ }
+ }
+ if !reflect.DeepEqual(types, tc.types) {
+ t.Fatalf("content types: got %v want %v", types, tc.types)
+ }
+ })
+ }
+}