Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(agent): support real streaming in canvas LLM component (#21468)

Sanderhoff-alt1133b9fd39f4
diff --git a/internal/agent/canvas/canvas_streaming_e2e_test.go b/internal/agent/canvas/canvas_streaming_e2e_test.go
new file mode 100644
index 0000000000..4c124779ab
--- /dev/null
+++ b/internal/agent/canvas/canvas_streaming_e2e_test.go
@@ -0,0 +1,250 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package canvas
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "strings"
+ "sync"
+ "testing"
+
+ "gorm.io/gorm"
+ "ragflow/internal/agent/chat"
+ "ragflow/internal/agent/component"
+ "ragflow/internal/agent/runtime"
+)
+
+type mockCanvasStreamingInvoker struct {
+ mu sync.Mutex
+ calls int
+ deltas []string
+ postErr error
+}
+
+func (m *mockCanvasStreamingInvoker) Invoke(ctx context.Context, db *gorm.DB, req chat.Request) (*chat.Response, error) {
+ m.mu.Lock()
+ m.calls++
+ m.mu.Unlock()
+ return &chat.Response{Content: "full"}, nil
+}
+
+func (m *mockCanvasStreamingInvoker) Stream(ctx context.Context, db *gorm.DB, req chat.Request, onDelta func(string, bool) error) (*chat.Response, error) {
+ m.mu.Lock()
+ m.calls++
+ m.mu.Unlock()
+ for _, d := range m.deltas {
+ if onDelta != nil {
+ if err := onDelta(d, false); err != nil {
+ return nil, err
+ }
+ }
+ }
+ if m.postErr != nil {
+ return nil, m.postErr
+ }
+ return &chat.Response{Content: strings.Join(m.deltas, "")}, nil
+}
+
+func TestCanvas_DeferredLLMStreaming_E2E(t *testing.T) {
+ makeStreamingCanvas := func(llmParams map[string]any) *Canvas {
+ return &Canvas{
+ Components: map[string]CanvasComponent{
+ "begin_0": {
+ Obj: CanvasComponentObj{ComponentName: "Begin", Params: map[string]any{}},
+ Downstream: []string{"llm_0"},
+ },
+ "llm_0": {
+ Obj: CanvasComponentObj{
+ ComponentName: "LLM",
+ Params: llmParams,
+ },
+ Upstream: []string{"begin_0"},
+ Downstream: []string{"message_0"},
+ },
+ "message_0": {
+ Obj: CanvasComponentObj{
+ ComponentName: "Message",
+ Params: map[string]any{
+ "text": "{llm_0@content}",
+ },
+ },
+ Upstream: []string{"llm_0"},
+ Downstream: []string{},
+ },
+ },
+ Path: []string{"begin_0", "llm_0", "message_0"},
+ }
+ }
+
+ t.Run("successful streaming delivers multiple deltas and updates node_finished outputs", func(t *testing.T) {
+ inv := &mockCanvasStreamingInvoker{deltas: []string{"Hello", " ", "streaming", " ", "world"}}
+ prev := component.GetDefaultChatInvokerForTest()
+ component.SetDefaultChatInvoker(inv)
+ t.Cleanup(func() { component.SetDefaultChatInvoker(prev) })
+
+ c := makeStreamingCanvas(map[string]any{
+ "model_id": "test-model@openai",
+ "user_prompt": "hello",
+ })
+ cc, err := Compile(t.Context(), c)
+ if err != nil {
+ t.Fatalf("Compile() failed: %v", err)
+ }
+
+ runState := NewCanvasState("run-e2e-ok", "task-e2e-ok")
+ ctx := withState(t.Context(), runState)
+ ctx = runtime.WithDeferredNodeRegistry(ctx)
+
+ var emittedDeltas []string
+ ctx = runtime.WithCanvasMessageEventEmitter(ctx, func(content string, startToThink, endToThink bool) {
+ if content != "" {
+ emittedDeltas = append(emittedDeltas, content)
+ }
+ })
+
+ eventsChan := make(chan RunEvent, 100)
+ ctx = WithRunMeta(ctx, &RunMeta{
+ Events: eventsChan,
+ MessageID: "msg-1",
+ SessionID: "sess-1",
+ })
+
+ _, err = cc.Workflow.Invoke(ctx, map[string]any{"query": "go"})
+ if err != nil {
+ t.Fatalf("Workflow.Invoke() unexpected err = %v", err)
+ }
+
+ if inv.calls != 1 {
+ t.Errorf("LLM invoker calls = %d, want 1", inv.calls)
+ }
+ if len(emittedDeltas) != len(inv.deltas) {
+ t.Errorf("emitted deltas count = %d, want %d (%v)", len(emittedDeltas), len(inv.deltas), emittedDeltas)
+ }
+ if strings.Join(emittedDeltas, "") != "Hello streaming world" {
+ t.Errorf("emitted text = %q, want %q", strings.Join(emittedDeltas, ""), "Hello streaming world")
+ }
+
+ // Verify node_finished event for llm_0
+ var llmFinished *NodeFinishedData
+ close(eventsChan)
+ for ev := range eventsChan {
+ if ev.Type == "node_finished" {
+ var nf NodeFinishedData
+ if err := json.Unmarshal([]byte(ev.Data), &nf); err == nil && nf.ComponentID == "llm_0" {
+ llmFinished = &nf
+ }
+ }
+ }
+
+ if llmFinished == nil {
+ t.Fatal("no node_finished event found for llm_0")
+ }
+ if llmFinished.Error != nil {
+ t.Errorf("node_finished Error = %v, want nil", llmFinished.Error)
+ }
+ outputs, _ := llmFinished.Outputs.(map[string]any)
+ if content, ok := outputs["content"].(string); !ok || content != "Hello streaming world" {
+ t.Errorf("node_finished Outputs[content] = %v, want %q", outputs["content"], "Hello streaming world")
+ }
+ })
+
+ t.Run("mid-stream failure surfaces DeferredStreamError and records error in node_finished", func(t *testing.T) {
+ streamErr := errors.New("abrupt stream termination")
+ inv := &mockCanvasStreamingInvoker{
+ deltas: []string{"chunk 1", "chunk 2"},
+ postErr: streamErr,
+ }
+ prev := component.GetDefaultChatInvokerForTest()
+ component.SetDefaultChatInvoker(inv)
+ t.Cleanup(func() { component.SetDefaultChatInvoker(prev) })
+
+ c := makeStreamingCanvas(map[string]any{
+ "model_id": "test-model@openai",
+ "user_prompt": "hello",
+ })
+ cc, err := Compile(t.Context(), c)
+ if err != nil {
+ t.Fatalf("Compile() failed: %v", err)
+ }
+
+ runState := NewCanvasState("run-e2e-err", "task-e2e-err")
+ ctx := withState(t.Context(), runState)
+ ctx = runtime.WithDeferredNodeRegistry(ctx)
+
+ eventsChan := make(chan RunEvent, 100)
+ ctx = WithRunMeta(ctx, &RunMeta{
+ Events: eventsChan,
+ MessageID: "msg-err",
+ SessionID: "sess-err",
+ })
+
+ _, err = cc.Workflow.Invoke(ctx, map[string]any{"query": "go"})
+ if err == nil {
+ t.Fatal("expected Workflow.Invoke to return error on mid-stream failure, got nil")
+ }
+
+ var deferredErr *runtime.DeferredStreamError
+ if !errors.As(err, &deferredErr) && !strings.Contains(err.Error(), "abrupt stream termination") {
+ t.Errorf("err = %v, want DeferredStreamError or stream termination text", err)
+ }
+
+ if inv.calls != 1 {
+ t.Errorf("LLM invoker calls = %d, want 1", inv.calls)
+ }
+
+ // Verify node_finished event for llm_0 reports error
+ var llmFinished *NodeFinishedData
+ close(eventsChan)
+ for ev := range eventsChan {
+ if ev.Type == "node_finished" {
+ var nf NodeFinishedData
+ if err := json.Unmarshal([]byte(ev.Data), &nf); err == nil && nf.ComponentID == "llm_0" {
+ llmFinished = &nf
+ }
+ }
+ }
+
+ if llmFinished == nil {
+ t.Fatal("no node_finished event found for llm_0")
+ }
+ if llmFinished.Error == nil || !strings.Contains(llmFinished.Error.(string), "abrupt stream termination") {
+ t.Errorf("node_finished Error = %v, want abrupt stream termination", llmFinished.Error)
+ }
+ })
+
+ t.Run("LLM with json_output or output_structure gates to eager execution", func(t *testing.T) {
+ cases := []struct {
+ name string
+ params map[string]any
+ }{
+ {"json_output=true", map[string]any{"model_id": "test-model@openai", "json_output": true}},
+ {"output_structure set", map[string]any{"model_id": "test-model@openai", "output_structure": map[string]any{"reply": "str"}}},
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ c := makeStreamingCanvas(tc.params)
+ opts := resolveComponentExecutionOptions(c, "llm_0", "LLM")
+ if opts.DeferLLMToMessage {
+ t.Errorf("expected DeferLLMToMessage=false for %s, got true", tc.name)
+ }
+ })
+ }
+ })
+}
diff --git a/internal/agent/canvas/direct_message_downstream_test.go b/internal/agent/canvas/direct_message_downstream_test.go
index 0e213d3ee3..e51063ac3f 100644
--- a/internal/agent/canvas/direct_message_downstream_test.go
+++ b/internal/agent/canvas/direct_message_downstream_test.go
@@ -1,7 +1,11 @@
// Package canvas — directMessageDownstream unit tests.
package canvas
-import "testing"
+import (
+ "testing"
+
+ "ragflow/internal/agent/runtime"
+)
// TestDirectMessageDownstream covers the mixed-downstream contract:
// deferral is only allowed when every direct downstream is a terminal Message.
@@ -198,6 +202,57 @@ func TestDirectMessageDownstream(t *testing.T) {
cpnID: "agent_0",
deferred: true,
},
+ {
+ name: "LLM component with sole message downstream defers",
+ canvas: &Canvas{
+ Components: map[string]CanvasComponent{
+ "llm_0": {
+ Obj: CanvasComponentObj{ComponentName: "LLM"},
+ Downstream: []string{"message_0"},
+ },
+ "message_0": {
+ Obj: CanvasComponentObj{ComponentName: "Message"},
+ Downstream: []string{},
+ },
+ },
+ },
+ cpnID: "llm_0",
+ deferred: true,
+ },
+ {
+ name: "Generate component with sole message downstream defers",
+ canvas: &Canvas{
+ Components: map[string]CanvasComponent{
+ "gen_0": {
+ Obj: CanvasComponentObj{ComponentName: "Generate"},
+ Downstream: []string{"message_0"},
+ },
+ "message_0": {
+ Obj: CanvasComponentObj{ComponentName: "Message"},
+ Downstream: []string{},
+ },
+ },
+ },
+ cpnID: "gen_0",
+ deferred: true,
+ },
+ {
+ name: "LLM component followed by non-message does not defer",
+ canvas: &Canvas{
+ Components: map[string]CanvasComponent{
+ "llm_0": {
+ Obj: CanvasComponentObj{ComponentName: "LLM"},
+ Downstream: []string{"sql_0"},
+ },
+ "sql_0": {
+ Obj: CanvasComponentObj{ComponentName: "ExeSQL"},
+ Downstream: []string{},
+ },
+ },
+ },
+ cpnID: "llm_0",
+ deferred: false,
+ },
}
for _, tc := range cases {
@@ -208,3 +263,54 @@ func TestDirectMessageDownstream(t *testing.T) {
})
}
}
+
+func TestResolveComponentExecutionOptions(t *testing.T) {
+ makeCanvas := func(name string, downstream []string, msgParams map[string]any) *Canvas {
+ c := &Canvas{
+ Components: map[string]CanvasComponent{
+ "root": {Obj: CanvasComponentObj{ComponentName: name}, Downstream: downstream},
+ },
+ }
+ for _, d := range downstream {
+ c.Components[d] = CanvasComponent{
+ Obj: CanvasComponentObj{ComponentName: "Message", Params: msgParams},
+ }
+ }
+ return c
+ }
+
+ cases := []struct {
+ name string
+ canvas *Canvas
+ cpnName string
+ wantOpts runtime.ComponentExecutionOptions
+ }{
+ {"LLM consuming via text", makeCanvas("LLM", []string{"m0"}, map[string]any{"text": "Ans: {root@content}"}), "LLM", runtime.ComponentExecutionOptions{DeferLLMToMessage: true}},
+ {"LLM with json_output disables deferral", &Canvas{Components: map[string]CanvasComponent{
+ "root": {Obj: CanvasComponentObj{ComponentName: "LLM", Params: map[string]any{"json_output": true}}, Downstream: []string{"m0"}},
+ "m0": {Obj: CanvasComponentObj{ComponentName: "Message", Params: map[string]any{"text": "{root@content}"}}},
+ }}, "LLM", runtime.ComponentExecutionOptions{}},
+ {"LLM with output_structure disables deferral", &Canvas{Components: map[string]CanvasComponent{
+ "root": {Obj: CanvasComponentObj{ComponentName: "LLM", Params: map[string]any{"output_structure": map[string]any{"key": "val"}}}, Downstream: []string{"m0"}},
+ "m0": {Obj: CanvasComponentObj{ComponentName: "Message", Params: map[string]any{"text": "{root@content}"}}},
+ }}, "LLM", runtime.ComponentExecutionOptions{}},
+ {"Generate consuming via content", makeCanvas("Generate", []string{"m0"}, map[string]any{"content": []any{"{{root@content}}"}}), "Generate", runtime.ComponentExecutionOptions{DeferLLMToMessage: true}},
+ {"LLM constant template", makeCanvas("LLM", []string{"m0"}, map[string]any{"text": "All done!"}), "LLM", runtime.ComponentExecutionOptions{}},
+ {"LLM referencing other node", makeCanvas("LLM", []string{"m0"}, map[string]any{"text": "{sql_0@result}"}), "LLM", runtime.ComponentExecutionOptions{}},
+ {"LLM multiple messages", makeCanvas("LLM", []string{"m0", "m1"}, map[string]any{"text": "{root@content}"}), "LLM", runtime.ComponentExecutionOptions{}},
+ {"Agent multiple messages", makeCanvas("Agent", []string{"m0", "m1"}, nil), "Agent", runtime.ComponentExecutionOptions{DeferAgentToMessage: true}},
+ {"Agent without message downstream", &Canvas{Components: map[string]CanvasComponent{
+ "root": {Obj: CanvasComponentObj{ComponentName: "Agent"}, Downstream: []string{"sql_0"}},
+ "sql_0": {Obj: CanvasComponentObj{ComponentName: "ExeSQL"}},
+ }}, "Agent", runtime.ComponentExecutionOptions{SuppressAgentMessageEvents: true}},
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ got := resolveComponentExecutionOptions(tc.canvas, "root", tc.cpnName)
+ if got != tc.wantOpts {
+ t.Errorf("resolveComponentExecutionOptions() = %+v, want %+v", got, tc.wantOpts)
+ }
+ })
+ }
+}
diff --git a/internal/agent/canvas/loop_subgraph.go b/internal/agent/canvas/loop_subgraph.go
index 36c3a717cd..57129ec6b5 100644
--- a/internal/agent/canvas/loop_subgraph.go
+++ b/internal/agent/canvas/loop_subgraph.go
@@ -42,7 +42,6 @@ import (
"slices"
"strings"
- "ragflow/internal/agent/runtime"
"ragflow/internal/agent/workflowx"
"github.com/cloudwego/eino/compose"
@@ -255,11 +254,7 @@ func buildSubWorkflow(
if name == "" {
return nil, fmt.Errorf("agent: loop %q member %q has empty component_name", loopID, cpnID)
}
- deferToMessage := directMessageDownstream(c, cpnID)
- nodeOpts := runtime.ComponentExecutionOptions{
- DeferAgentToMessage: deferToMessage,
- SuppressAgentMessageEvents: strings.EqualFold(name, "Agent") && !deferToMessage,
- }
+ nodeOpts := resolveComponentExecutionOptions(c, cpnID, name)
body, err := buildNodeBodyWithOptions(ctx, cpnID, name, c.Components[cpnID].DisplayName, c.Components[cpnID].Obj.Params, nodeOpts)
if err != nil {
return nil, err
diff --git a/internal/agent/canvas/node_body.go b/internal/agent/canvas/node_body.go
index 6eaa4c3dc1..602e1071c2 100644
--- a/internal/agent/canvas/node_body.go
+++ b/internal/agent/canvas/node_body.go
@@ -369,8 +369,8 @@ func withStateBracket(cpnID, componentName string, body nodeBodyFn) nodeBodyFn {
state.SetVar(outputCpnID, k, v)
}
if runtime.IsDeferredStream(out["content"]) {
- runtime.RegisterDeferredNode(ctx, cpnID, func() {
- nodeFinishedNow(ctx, state, cpnID, componentName, componentName, nil)
+ runtime.RegisterDeferredNode(ctx, cpnID, func(nodeErr error) {
+ nodeFinishedNow(ctx, state, cpnID, componentName, componentName, nodeErr)
})
} else {
nodeFinishedNow(ctx, state, cpnID, componentName, componentName, nil)
diff --git a/internal/agent/canvas/scheduler.go b/internal/agent/canvas/scheduler.go
index de35820aeb..a150dba442 100644
--- a/internal/agent/canvas/scheduler.go
+++ b/internal/agent/canvas/scheduler.go
@@ -325,9 +325,18 @@ func nodeFinishedNow(ctx context.Context, state *CanvasState, cpnID, componentNa
}
// Collect outputs from the state's Outputs bucket for this cpn.
+ // Read from runtime.GetStateFromContext(ctx) first so resolved outputs from Message
+ // take precedence over raw deferred streams.
var outputs map[string]any
- if bucket := state.Snapshot()[cpnID]; len(bucket) > 0 {
- outputs = bucket
+ if ctxState, _ := runtime.GetStateFromContext(ctx); ctxState != nil {
+ if bucket := ctxState.Snapshot()[cpnID]; len(bucket) > 0 {
+ outputs = bucket
+ }
+ }
+ if len(outputs) == 0 {
+ if bucket := state.Snapshot()[cpnID]; len(bucket) > 0 {
+ outputs = bucket
+ }
}
inputs := map[string]any{}
@@ -554,11 +563,7 @@ func BuildWorkflow(ctx context.Context, c *Canvas) (*compose.Workflow[map[string
if name == "" {
return nil, fmt.Errorf("agent: component %q has empty component_name", cpnID)
}
- deferToMessage := directMessageDownstream(c, cpnID)
- nodeOpts := runtime.ComponentExecutionOptions{
- DeferAgentToMessage: deferToMessage,
- SuppressAgentMessageEvents: strings.EqualFold(name, "Agent") && !deferToMessage,
- }
+ nodeOpts := resolveComponentExecutionOptions(c, cpnID, name)
body, err := buildNodeBodyWithOptions(ctx, cpnID, name, c.Components[cpnID].DisplayName, c.Components[cpnID].Obj.Params, nodeOpts)
if err != nil {
return nil, err
@@ -580,8 +585,12 @@ func BuildWorkflow(ctx context.Context, c *Canvas) (*compose.Workflow[map[string
// Python keeps the Agent node pending while Message consumes its
// partial generator. Message completes this callback after the
// deferred stream closes.
- runtime.RegisterDeferredNode(ctx, cpnID, func() {
- nodeFinishedNow(ctx, state, cpnID, componentName, componentName, nil)
+ runtime.RegisterDeferredNode(ctx, cpnID, func(nodeErr error) {
+ targetState := state
+ if ctxState, _ := runtime.GetStateFromContext(ctx); ctxState != nil {
+ targetState = ctxState
+ }
+ nodeFinishedNow(ctx, targetState, cpnID, componentName, componentName, nodeErr)
})
} else {
nodeFinishedNow(ctx, state, cpnID, componentName, componentName, postErr)
@@ -795,6 +804,85 @@ func directMessageDownstream(c *Canvas, cpnID string) bool {
return true
}
+func messageConsumesComponent(c *Canvas, msgID, cpnID string) bool {
+ if c == nil {
+ return false
+ }
+ msgComp, ok := c.Components[msgID]
+ if !ok || len(msgComp.Obj.Params) == 0 {
+ return false
+ }
+ needle := strings.ToLower(cpnID) + "@"
+ for _, v := range msgComp.Obj.Params {
+ switch x := v.(type) {
+ case string:
+ if strings.Contains(strings.ToLower(x), needle) {
+ return true
+ }
+ case []string:
+ for _, s := range x {
+ if strings.Contains(strings.ToLower(s), needle) {
+ return true
+ }
+ }
+ case []any:
+ for _, item := range x {
+ if s, ok := item.(string); ok && strings.Contains(strings.ToLower(s), needle) {
+ return true
+ }
+ }
+ }
+ }
+ return false
+}
+
+func hasStructuredOutput(params map[string]any) bool {
+ if params == nil {
+ return false
+ }
+ if v, ok := params["json_output"]; ok {
+ if b, ok := v.(bool); ok && b {
+ return true
+ }
+ if s, ok := v.(string); ok && strings.EqualFold(s, "true") {
+ return true
+ }
+ }
+ if v, ok := params["output_structure"]; ok && v != nil {
+ switch m := v.(type) {
+ case map[string]any:
+ if len(m) > 0 {
+ return true
+ }
+ case map[string]string:
+ if len(m) > 0 {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+func resolveComponentExecutionOptions(c *Canvas, cpnID string, name string) runtime.ComponentExecutionOptions {
+ deferToMessage := directMessageDownstream(c, cpnID)
+ isAgent := strings.EqualFold(name, "Agent")
+ isLLM := strings.EqualFold(name, "LLM") || strings.EqualFold(name, "Generate")
+ deferLLM := false
+ if isLLM && deferToMessage && c != nil {
+ comp, ok := c.Components[cpnID]
+ if ok && len(comp.Downstream) == 1 && messageConsumesComponent(c, comp.Downstream[0], cpnID) {
+ if !hasStructuredOutput(comp.Obj.Params) {
+ deferLLM = true
+ }
+ }
+ }
+ return runtime.ComponentExecutionOptions{
+ DeferAgentToMessage: deferToMessage && isAgent,
+ DeferLLMToMessage: deferLLM,
+ SuppressAgentMessageEvents: isAgent && !deferToMessage,
+ }
+}
+
func wireWorkflowTerminals(
wf *compose.Workflow[map[string]any, map[string]any],
terminals []string,
diff --git a/internal/agent/component/llm.go b/internal/agent/component/llm.go
index f920dd2797..99bd3b3af0 100644
--- a/internal/agent/component/llm.go
+++ b/internal/agent/component/llm.go
@@ -19,6 +19,7 @@ import (
"slices"
"sort"
"strings"
+ "sync"
"time"
"github.com/cloudwego/eino/schema"
@@ -681,8 +682,116 @@ func NewLLMComponent(p LLMParam) *LLMComponent {
// Name returns the registered component name.
func (c *LLMComponent) Name() string { return "LLM" }
-// Invoke runs the LLM and returns the output map.
+const defaultLLMDeferredTimeout = 10 * time.Minute
+
+func (c *LLMComponent) hasStructuredOutput(inputs map[string]any) bool {
+ if c.param.JSONOutput || len(c.param.OutputStructure) > 0 {
+ return true
+ }
+ if v, ok := inputs["json_output"]; ok {
+ if b, ok := v.(bool); ok && b {
+ return true
+ }
+ }
+ if v, ok := inputs["output_structure"]; ok && v != nil {
+ return true
+ }
+ return false
+}
+
+// Invoke runs the LLM component. When the component directly precedes a
+// terminal Message node in the canvas, it returns a lazy DeferredStream closure
+// so that Message can stream deltas in real-time. Otherwise, it executes eagerly.
func (c *LLMComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {
+ if runtime.ComponentExecutionOptionsFromContext(ctx).DeferLLMToMessage && !c.hasStructuredOutput(inputs) {
+ timeout := defaultLLMDeferredTimeout
+ if deadline, ok := ctx.Deadline(); ok {
+ if remaining := time.Until(deadline); remaining > 0 {
+ timeout = remaining
+ }
+ }
+ var once sync.Once
+ var resp map[string]any
+ var err error
+ deferred := &runtime.DeferredStream{
+ Open: func(openCtx context.Context, sink runtime.AgentDeltaSink) (map[string]any, error) {
+ once.Do(func() {
+ agentCtx, cancel := context.WithTimeout(openCtx, timeout)
+ defer cancel()
+ resp, err = c.invokeStreamNow(runtime.WithAgentDeltaSink(agentCtx, sink), db, inputs, sink)
+ })
+ return resp, err
+ },
+ }
+ return map[string]any{"content": deferred}, nil
+ }
+ return c.invokeNow(ctx, db, inputs)
+}
+
+func (c *LLMComponent) invokeNow(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {
+ inv, req, p, earlyOut, err := c.prepareChatRequest(ctx, db, inputs)
+ if err != nil {
+ return nil, err
+ }
+ if earlyOut != nil {
+ return earlyOut, nil
+ }
+ resp, err := inv.Invoke(ctx, db, req)
+ if err != nil {
+ return nil, fmt.Errorf("component: LLM.Invoke: %w", err)
+ }
+ return c.postProcessResponse(ctx, db, inv, req, p, resp)
+}
+
+func (c *LLMComponent) invokeStreamNow(ctx context.Context, db *gorm.DB, inputs map[string]any, sink runtime.AgentDeltaSink) (map[string]any, error) {
+ inv, req, p, earlyOut, err := c.prepareChatRequest(ctx, db, inputs)
+ if err != nil {
+ return nil, err
+ }
+ if earlyOut != nil {
+ if content, ok := earlyOut["content"].(string); ok && content != "" && sink != nil {
+ sink(content, "")
+ }
+ return earlyOut, nil
+ }
+
+ var resp *ChatInvokeResponse
+ if streamer, ok := inv.(chat.StreamingInvoker); ok {
+ resp, err = streamer.Stream(ctx, db, req, func(delta string, isThink bool) error {
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+ if sink != nil {
+ if isThink {
+ sink("", delta)
+ } else {
+ sink(delta, "")
+ }
+ }
+ return nil
+ })
+ } else {
+ resp, err = inv.Invoke(ctx, db, req)
+ if err == nil && resp != nil && sink != nil {
+ if resp.Thinking != "" {
+ sink("", resp.Thinking)
+ }
+ if resp.Content != "" {
+ sink(resp.Content, "")
+ }
+ }
+ }
+ if err != nil {
+ return nil, fmt.Errorf("component: LLM.Invoke: %w", err)
+ }
+ return c.postProcessResponse(ctx, db, inv, req, p, resp)
+}
+
+func (c *LLMComponent) prepareChatRequest(
+ ctx context.Context,
+ db *gorm.DB,
+ inputs map[string]any,
+) (ChatInvoker, ChatInvokeRequest, LLMParam, map[string]any, error) {
p := mergeLLMParam(c.param, inputs)
// Resolve tenant-scoped custom models (and fill missing driver/credentials)
@@ -693,7 +802,7 @@ func (c *LLMComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[strin
originalModelID := p.ModelID
p.ModelID, p.Driver, p.APIKey, p.BaseURL, err = resolveChatModelRef(ctx, db, p.ModelID, p.Driver, p.APIKey, p.BaseURL)
if err != nil {
- return nil, fmt.Errorf("component: LLM.Invoke: resolve model: %w", err)
+ return nil, ChatInvokeRequest{}, p, nil, fmt.Errorf("component: LLM.Invoke: resolve model: %w", err)
}
// Resolve the model's context window (content_length) for message
// fitting. 0 means the model is unknown → chat.FitMessages falls back to
@@ -717,10 +826,10 @@ func (c *LLMComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[strin
zap.String("model_name", p.ModelID))
}
if p.ModelID == "" {
- return nil, &ParamError{Field: "model_id", Reason: "required"}
+ return nil, ChatInvokeRequest{}, p, nil, &ParamError{Field: "model_id", Reason: "required"}
}
if p.UserPrompt == "" && p.SystemPrompt == "" {
- return nil, &ParamError{Field: "user_prompt", Reason: "at least one of user_prompt or system_prompt must be set"}
+ return nil, ChatInvokeRequest{}, p, nil, &ParamError{Field: "user_prompt", Reason: "at least one of user_prompt or system_prompt must be set"}
}
// Resolve {{cpn_id@var}} references in the system and user
// prompts against the canvas state attached to ctx. When the
@@ -767,7 +876,7 @@ func (c *LLMComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[strin
hasSysFilesPlaceholder := strings.Contains(p.SystemPrompt, "{sys.files}") || strings.Contains(p.UserPrompt, "{sys.files}")
if state, err := runtime.GetStateFromContext(ctx); err == nil && state != nil {
if err := rejectUnsupportedImages(ctx, db, state, originalModelID, p.ModelID); err != nil {
- return nil, err
+ return nil, ChatInvokeRequest{}, p, nil, err
}
sysFileTexts, sysFileImgs = collectSysFiles(state)
if len(sysFileImgs) > 0 {
@@ -840,7 +949,7 @@ func (c *LLMComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[strin
// does not duplicate it.
fitted, fitErr := chat.FitMessages("", msgs, contentLength)
if fitErr != "" {
- return map[string]any{"content": fitErr}, nil
+ return nil, ChatInvokeRequest{}, p, map[string]any{"content": fitErr}, nil
}
msgs = fitted
}
@@ -850,30 +959,6 @@ func (c *LLMComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[strin
// for a per-call retry budget. We RE-WRAP the default
// invoker in a fresh retryInvoker that respects those
// values literally.
- //
- // LLM retry normal-absolute-count: when MaxRetries OR
- // DelayAfterError is explicitly set on LLMParam, the
- // operator's intent is an ABSOLUTE attempt budget. The
- // default invoker installed at boot in cmd/server_main.go
- // is itself a retryInvoker wrapping einoChatInvoker.
- // Without unwrapping, the two loops would multiplicatively
- // stack:
- //
- // boot=3, MaxRetries=5 → up to (3+1) × (5+1) = 24
- // invocations, not the 6 the
- // operator almost certainly intended.
- //
- // unwrapChatInvoker peels off any retryInvoker layers to
- // reach the bare invoker, then the param-override branch
- // wraps that bare invoker in a fresh retryInvoker with the
- // operator's literal values. Net effect: the absolute attempt
- // count is exactly (MaxRetries + 1), independent of the boot
- // layer.
- //
- // Operators who do NOT set MaxRetries (both fields zero) get
- // the boot retry chain unchanged. The unit tests in
- // llm_retry_test.go pin both the unwrap behaviour and the
- // stacking-prevention
hasOverride := p.MaxRetries > 0 || p.DelayAfterError > 0
if hasOverride {
maxRetries := p.MaxRetries
@@ -881,12 +966,9 @@ func (c *LLMComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[strin
if delay <= 0 {
delay = retryInvokerBackoff
}
- // Normalise the attempt budget: peel off the boot
- // retryInvoker layer (if any) so the operator's
- // MaxRetries is an absolute count, not a stacked one.
inv = newRetryInvoker(unwrapChatInvoker(inv), maxRetries, delay)
}
- resp, err := inv.Invoke(ctx, db, ChatInvokeRequest{
+ req := ChatInvokeRequest{
Driver: p.Driver,
ModelName: p.ModelID,
APIKey: p.APIKey,
@@ -898,11 +980,18 @@ func (c *LLMComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[strin
FrequencyPenalty: p.FrequencyPenalty,
MaxTokens: p.MaxTokens,
Thinking: p.Thinking,
- })
- if err != nil {
- return nil, fmt.Errorf("component: LLM.Invoke: %w", err)
}
+ return inv, req, p, nil, nil
+}
+func (c *LLMComponent) postProcessResponse(
+ ctx context.Context,
+ db *gorm.DB,
+ inv ChatInvoker,
+ req ChatInvokeRequest,
+ p LLMParam,
+ resp *ChatInvokeResponse,
+) (map[string]any, error) {
// Strip think blocks + JSON fences from the response.
// Mirrors Python's clean_formated_answer() exactly
// (re.sub(r"^.*</think>", "", ...) + ^.*```json + trailing ```).
@@ -986,24 +1075,17 @@ func (c *LLMComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[strin
// downstream consumers can flush state without relying on channel close
// alone (close also works; the "done" key is informational).
//
-// Today, the LLM driver layer returns a single non-streamed response,
-// so this v1 emits exactly one chunk + one done. Hooking the actual
-// eino stream (EinoChatModel.Stream at internal/entity/models/llm.go:137)
-// is deferred — the public surface here is correct, only the data
-// source needs to be swapped to a real StreamReader consumer in a
-// follow-up.
+// Stream satisfies Component.Stream: it emits reasoning and content deltas as
+// they arrive from the model, followed by a final done marker.
func (c *LLMComponent) Stream(ctx context.Context, db *gorm.DB, inputs map[string]any) (<-chan map[string]any, error) {
out := make(chan map[string]any, 16)
go func() {
defer close(out)
- // Early bail-out for pre-cancelled contexts: don't run the
- // (potentially expensive) LLM call when the consumer has
- // already given up. Honors the documented select-on-ctx
- // pattern at the goroutine entry, not just between chunks.
if err := ctx.Err(); err != nil {
return
}
- result, err := c.Invoke(ctx, db, inputs)
+
+ inv, req, _, earlyOut, err := c.prepareChatRequest(ctx, db, inputs)
if err != nil {
select {
case out <- map[string]any{"error": err.Error()}:
@@ -1011,21 +1093,72 @@ func (c *LLMComponent) Stream(ctx context.Context, db *gorm.DB, inputs map[strin
}
return
}
- // Single non-streamed response — emit as one content chunk.
- // A real streaming integration would loop over a channel
- // here and emit multiple chunks with partial content.
- chunk := map[string]any{
- "thinking": result["thinking"],
- "content": result["content"],
+ if earlyOut != nil {
+ if content, ok := earlyOut["content"].(string); ok && content != "" {
+ select {
+ case out <- map[string]any{"content": content}:
+ case <-ctx.Done():
+ return
+ }
+ }
+ select {
+ case out <- map[string]any{"done": true}:
+ case <-ctx.Done():
+ }
+ return
}
- select {
- case out <- chunk:
- case <-ctx.Done():
+
+ var resp *ChatInvokeResponse
+ if streamer, ok := inv.(chat.StreamingInvoker); ok {
+ resp, err = streamer.Stream(ctx, db, req, func(delta string, isThink bool) error {
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+ if delta == "" {
+ return nil
+ }
+ chunk := map[string]any{}
+ if isThink {
+ chunk["thinking"] = delta
+ } else {
+ chunk["content"] = delta
+ }
+ select {
+ case out <- chunk:
+ return nil
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+ })
+ } else {
+ resp, err = inv.Invoke(ctx, db, req)
+ if err == nil && resp != nil {
+ chunk := map[string]any{
+ "thinking": resp.Thinking,
+ "content": resp.Content,
+ }
+ select {
+ case out <- chunk:
+ case <-ctx.Done():
+ return
+ }
+ }
+ }
+
+ if err != nil {
+ select {
+ case out <- map[string]any{"error": err.Error()}:
+ case <-ctx.Done():
+ }
return
}
- // Final done marker.
+
+ modelName := ""
+ if resp != nil {
+ modelName = resp.Model
+ }
select {
- case out <- map[string]any{"done": true, "model": result["model"]}:
+ case out <- map[string]any{"done": true, "model": modelName}:
case <-ctx.Done():
}
}()
diff --git a/internal/agent/component/llm_retry.go b/internal/agent/component/llm_retry.go
index 1b3b3fe651..0b2d2deb2a 100644
--- a/internal/agent/component/llm_retry.go
+++ b/internal/agent/component/llm_retry.go
@@ -26,6 +26,7 @@ import (
"fmt"
"time"
+ "ragflow/internal/agent/chat"
"ragflow/internal/common"
"gorm.io/gorm"
@@ -134,3 +135,65 @@ func (r *retryInvoker) Invoke(ctx context.Context, db *gorm.DB, req ChatInvokeRe
}
return resp, nil
}
+
+// Stream satisfies chat.StreamingInvoker. It delegates to the inner invoker
+// when it implements chat.StreamingInvoker, or falls back to Invoke.
+func (r *retryInvoker) Stream(ctx context.Context, db *gorm.DB, req ChatInvokeRequest, onDelta func(delta string, isThink bool) error) (*ChatInvokeResponse, error) {
+ if r == nil || r.inner == nil {
+ return nil, fmt.Errorf("component: retryInvoker: nil inner")
+ }
+ if onDelta == nil {
+ onDelta = func(string, bool) error { return nil }
+ }
+
+ streamer, ok := r.inner.(chat.StreamingInvoker)
+ if !ok {
+ resp, err := r.Invoke(ctx, db, req)
+ if err != nil {
+ return nil, err
+ }
+ if resp != nil {
+ if resp.Thinking != "" {
+ if err := onDelta(resp.Thinking, true); err != nil {
+ return nil, err
+ }
+ }
+ if resp.Content != "" {
+ if err := onDelta(resp.Content, false); err != nil {
+ return nil, err
+ }
+ }
+ }
+ return resp, nil
+ }
+
+ var resp *ChatInvokeResponse
+ var callbackErr error
+ emitted := false
+ err := common.RetryWithBackoff(ctx, r.maxRetries, r.initialDelay, func() error {
+ callbackErr = nil
+ var streamErr error
+ resp, streamErr = streamer.Stream(ctx, db, req, func(delta string, isThink bool) error {
+ if delta != "" {
+ emitted = true
+ }
+ if err := onDelta(delta, isThink); err != nil {
+ callbackErr = err
+ return err
+ }
+ return nil
+ })
+ if callbackErr != nil {
+ return callbackErr
+ }
+ return streamErr
+ }, func(error) bool {
+ return callbackErr == nil && !emitted
+ })
+ if err != nil {
+ return nil, err
+ }
+ return resp, nil
+}
+
+var _ chat.StreamingInvoker = (*retryInvoker)(nil)
diff --git a/internal/agent/component/llm_retry_test.go b/internal/agent/component/llm_retry_test.go
index 8a263ae804..cf97a2bbd7 100644
--- a/internal/agent/component/llm_retry_test.go
+++ b/internal/agent/component/llm_retry_test.go
@@ -404,3 +404,79 @@ func TestLLM_NoParamOverride_StackingPreserved(t *testing.T) {
t.Errorf("bare.calls=%d, want %d (boot layer ran unchanged — no param override means we keep the implicit 3-retry budget)", got, want)
}
}
+
+type mockStreamingInvoker struct {
+ calls int32
+ failTimes int32
+ err error
+ deltas []string
+ postErr error
+}
+
+func (m *mockStreamingInvoker) Invoke(context.Context, *gorm.DB, ChatInvokeRequest) (*ChatInvokeResponse, error) {
+ return nil, errors.New("unexpected Invoke call")
+}
+
+func (m *mockStreamingInvoker) Stream(_ context.Context, _ *gorm.DB, _ ChatInvokeRequest, onDelta func(delta string, isThink bool) error) (*ChatInvokeResponse, error) {
+ n := atomic.AddInt32(&m.calls, 1)
+ if n <= atomic.LoadInt32(&m.failTimes) {
+ return nil, m.err
+ }
+ for _, d := range m.deltas {
+ if onDelta != nil {
+ if err := onDelta(d, false); err != nil {
+ return nil, err
+ }
+ }
+ }
+ return &ChatInvokeResponse{Content: strings.Join(m.deltas, "")}, m.postErr
+}
+
+func TestRetryInvoker_Stream(t *testing.T) {
+ t.Run("retries transient error before first delta", func(t *testing.T) {
+ inner := &mockStreamingInvoker{failTimes: 1, err: errors.New("blip"), deltas: []string{"o", "k"}}
+ inv := newRetryInvoker(inner, 2, time.Millisecond)
+
+ var got []string
+ resp, err := inv.Stream(t.Context(), nil, ChatInvokeRequest{}, func(d string, _ bool) error {
+ got = append(got, d)
+ return nil
+ })
+ if err != nil || resp == nil || resp.Content != "ok" || inner.calls != 2 || strings.Join(got, "") != "ok" {
+ t.Errorf("got %v, err %v, calls %d", resp, err, inner.calls)
+ }
+ })
+
+ t.Run("does not retry after deltas are emitted", func(t *testing.T) {
+ inner := &mockStreamingInvoker{deltas: []string{"partial"}, postErr: errors.New("mid-stream fail")}
+ inv := newRetryInvoker(inner, 3, time.Millisecond)
+
+ _, err := inv.Stream(t.Context(), nil, ChatInvokeRequest{}, nil)
+ if err == nil || inner.calls != 1 {
+ t.Errorf("expected error, inner.calls = %d, want 1", inner.calls)
+ }
+ })
+
+ t.Run("stops and propagates callback error without retrying", func(t *testing.T) {
+ inner := &mockStreamingInvoker{deltas: []string{"a", "b"}}
+ inv := newRetryInvoker(inner, 3, time.Millisecond)
+
+ sinkErr := errors.New("sink broken")
+ _, err := inv.Stream(t.Context(), nil, ChatInvokeRequest{}, func(string, bool) error { return sinkErr })
+ if !errors.Is(err, sinkErr) || inner.calls != 1 {
+ t.Errorf("err = %v, calls = %d, want sinkErr and 1 call", err, inner.calls)
+ }
+ })
+
+ t.Run("non-streaming invoker fallback propagates callback error", func(t *testing.T) {
+ resp := &ChatInvokeResponse{Thinking: "ponder", Content: "reply"}
+ inner := &scriptedInvoker{failTimes: 0, resp: resp}
+ inv := newRetryInvoker(inner, 1, time.Millisecond)
+
+ sinkErr := errors.New("sink failed")
+ _, err := inv.Stream(t.Context(), nil, ChatInvokeRequest{}, func(string, bool) error { return sinkErr })
+ if !errors.Is(err, sinkErr) {
+ t.Errorf("err = %v, want sinkErr", err)
+ }
+ })
+}
diff --git a/internal/agent/component/message.go b/internal/agent/component/message.go
index 1573434347..e912b7f0d8 100644
--- a/internal/agent/component/message.go
+++ b/internal/agent/component/message.go
@@ -470,7 +470,14 @@ func (m *MessageComponent) resolveDeferredTemplate(ctx context.Context, text str
if inThinking {
runtime.EmitCanvasMessageEvent(ctx, "", false, true)
}
+ targetNodeID := deferredNodeID
+ if targetNodeID == "" && strings.Contains(ref, "@") {
+ targetNodeID, _, _ = strings.Cut(ref, "@")
+ }
if err != nil {
+ if targetNodeID != "" {
+ runtime.CompleteDeferredNode(ctx, targetNodeID, err)
+ }
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return "", true, &runtime.DeferredStreamError{Err: err}
}
@@ -479,6 +486,9 @@ func (m *MessageComponent) resolveDeferredTemplate(ctx context.Context, text str
return "", true, &runtime.DeferredStreamError{Text: "**ERROR**: " + runtime.MarkdownSafeErrorText(err)}
}
if resultErr, _ := result["_ERROR"].(string); strings.TrimSpace(resultErr) != "" {
+ if targetNodeID != "" {
+ runtime.CompleteDeferredNode(ctx, targetNodeID, errors.New(resultErr))
+ }
return "", true, &runtime.DeferredStreamError{Text: resultErr}
}
finalText := visible.String()
@@ -487,14 +497,13 @@ func (m *MessageComponent) resolveDeferredTemplate(ctx context.Context, text str
finalText = completedContent
}
}
- if strings.Contains(ref, "@") {
- nodeID, _, _ := strings.Cut(ref, "@")
+ if targetNodeID != "" {
if result != nil {
for key, output := range result {
- state.SetVar(nodeID, key, output)
+ state.SetVar(targetNodeID, key, output)
}
}
- runtime.CompleteDeferredNode(ctx, nodeID)
+ runtime.CompleteDeferredNode(ctx, targetNodeID, nil)
}
resolved := runtime.ResolveTemplateForDisplay(text[start:end], state)
if strings.HasSuffix(ref, "@content") {
diff --git a/internal/agent/component/message_test.go b/internal/agent/component/message_test.go
index 864c22a736..10e8051d8b 100644
--- a/internal/agent/component/message_test.go
+++ b/internal/agent/component/message_test.go
@@ -425,14 +425,18 @@ func TestMessage_DeferredStreamSurfacesResultError(t *testing.T) {
})
ctx := runtime.WithDeferredNodeRegistry(withStateForTest(t.Context(), state))
completed := false
- runtime.RegisterDeferredNode(ctx, "agent_0", func() { completed = true })
+ var completedErr error
+ runtime.RegisterDeferredNode(ctx, "agent_0", func(err error) {
+ completed = true
+ completedErr = err
+ })
_, err := c.Invoke(ctx, nil, map[string]any{"text": "{{agent_0@content}}"})
if err == nil || !strings.Contains(err.Error(), "upstream 402") {
t.Fatalf("Invoke error = %v, want upstream 402", err)
}
- if completed {
- t.Fatal("deferred node completed after result _ERROR")
+ if !completed || completedErr == nil || !strings.Contains(completedErr.Error(), "upstream 402") {
+ t.Fatalf("expected deferred node completion with error, got completed=%v, err=%v", completed, completedErr)
}
}
diff --git a/internal/agent/component/streaming_test.go b/internal/agent/component/streaming_test.go
index e76c00ac9e..71d7795794 100644
--- a/internal/agent/component/streaming_test.go
+++ b/internal/agent/component/streaming_test.go
@@ -18,8 +18,13 @@ package component
import (
"context"
+ "sync/atomic"
"testing"
"time"
+
+ "ragflow/internal/agent/runtime"
+
+ "gorm.io/gorm"
)
// TestLLM_Stream_HappyPath: Stream emits content + done chunks, channel
@@ -135,3 +140,190 @@ func TestLLM_Stream_BufferDoesNotBlock(t *testing.T) {
t.Fatal("Stream did not drain within 2s — likely deadlock")
}
}
+
+type stubStreamingInvoker struct {
+ calls int32
+ deltas []string
+ thinks []string
+ resp *ChatInvokeResponse
+ err error
+}
+
+func (s *stubStreamingInvoker) Invoke(_ context.Context, _ *gorm.DB, _ ChatInvokeRequest) (*ChatInvokeResponse, error) {
+ atomic.AddInt32(&s.calls, 1)
+ if s.err != nil {
+ return nil, s.err
+ }
+ return s.resp, nil
+}
+
+func (s *stubStreamingInvoker) Stream(_ context.Context, _ *gorm.DB, _ ChatInvokeRequest, onDelta func(delta string, isThink bool) error) (*ChatInvokeResponse, error) {
+ atomic.AddInt32(&s.calls, 1)
+ if s.err != nil {
+ return nil, s.err
+ }
+ for _, th := range s.thinks {
+ if err := onDelta(th, true); err != nil {
+ return nil, err
+ }
+ }
+ for _, d := range s.deltas {
+ if err := onDelta(d, false); err != nil {
+ return nil, err
+ }
+ }
+ return s.resp, nil
+}
+
+func TestLLM_Stream_RealStreamingDeltas(t *testing.T) {
+ stub := &stubStreamingInvoker{
+ thinks: []string{"think1", "think2"},
+ deltas: []string{"hello ", "world"},
+ resp: &ChatInvokeResponse{Content: "hello world", Thinking: "think1think2", Model: "stream-model"},
+ }
+ withStubInvoker(t, stub)
+ ctx := t.Context()
+
+ c := NewLLMComponent(LLMParam{ModelID: "stream-model"})
+ ch, err := c.Stream(ctx, nil, map[string]any{"user_prompt": "hi"})
+ if err != nil {
+ t.Fatalf("Stream: %v", err)
+ }
+ var got []map[string]any
+ for chunk := range ch {
+ got = append(got, chunk)
+ }
+ if len(got) != 5 {
+ t.Fatalf("expected 5 chunks (2 thinks + 2 deltas + done), got %d: %+v", len(got), got)
+ }
+ if got[0]["thinking"] != "think1" || got[1]["thinking"] != "think2" {
+ t.Errorf("thinking deltas mismatch: %+v", got[:2])
+ }
+ if got[2]["content"] != "hello " || got[3]["content"] != "world" {
+ t.Errorf("content deltas mismatch: %+v", got[2:4])
+ }
+ if got[4]["done"] != true || got[4]["model"] != "stream-model" {
+ t.Errorf("done chunk mismatch: %+v", got[4])
+ }
+}
+
+func TestLLM_Invoke_DeferredStream(t *testing.T) {
+ stub := &stubStreamingInvoker{
+ thinks: []string{"thinking "},
+ deltas: []string{"hello ", "world"},
+ resp: &ChatInvokeResponse{Content: "hello world", Thinking: "thinking ", Model: "m"},
+ }
+ withStubInvoker(t, stub)
+ ctx := runtime.WithComponentExecutionOptions(t.Context(), runtime.ComponentExecutionOptions{
+ DeferLLMToMessage: true,
+ })
+
+ c := NewLLMComponent(LLMParam{ModelID: "m"})
+ out, err := c.Invoke(ctx, nil, map[string]any{"user_prompt": "hi"})
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ deferred, ok := out["content"].(*runtime.DeferredStream)
+ if !ok || deferred == nil {
+ t.Fatalf("expected *runtime.DeferredStream, got %T (%+v)", out["content"], out["content"])
+ }
+
+ var contentDeltas, thinkingDeltas []string
+ sink := func(contentDelta, thinkingDelta string) {
+ if contentDelta != "" {
+ contentDeltas = append(contentDeltas, contentDelta)
+ }
+ if thinkingDelta != "" {
+ thinkingDeltas = append(thinkingDeltas, thinkingDelta)
+ }
+ }
+
+ result, err := deferred.Open(t.Context(), sink)
+ if err != nil {
+ t.Fatalf("Open: %v", err)
+ }
+ if len(thinkingDeltas) != 1 || thinkingDeltas[0] != "thinking " {
+ t.Errorf("thinking deltas = %+v, want ['thinking ']", thinkingDeltas)
+ }
+ if len(contentDeltas) != 2 || contentDeltas[0] != "hello " || contentDeltas[1] != "world" {
+ t.Errorf("content deltas = %+v, want ['hello ', 'world']", contentDeltas)
+ }
+ if result["content"] != "hello world" {
+ t.Errorf("result content = %v, want hello world", result["content"])
+ }
+
+ // Verify idempotency: subsequent Open calls return cached result without re-invoking the model.
+ result2, err2 := deferred.Open(t.Context(), nil)
+ if err2 != nil {
+ t.Fatalf("second Open: %v", err2)
+ }
+ if result2["content"] != "hello world" {
+ t.Errorf("second Open result content = %v, want hello world", result2["content"])
+ }
+ if stub.calls != 1 {
+ t.Errorf("stub.calls = %d, want 1 (must be memoized via sync.Once)", stub.calls)
+ }
+}
+
+func TestLLM_Invoke_EagerWhenNotDeferred(t *testing.T) {
+ stub := &stubStreamingInvoker{
+ deltas: []string{"hello ", "world"},
+ resp: &ChatInvokeResponse{Content: "hello world", Model: "m"},
+ }
+ withStubInvoker(t, stub)
+ ctx := t.Context() // DeferLLMToMessage is false
+
+ c := NewLLMComponent(LLMParam{ModelID: "m"})
+ out, err := c.Invoke(ctx, nil, map[string]any{"user_prompt": "hi"})
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ content, ok := out["content"].(string)
+ if !ok || content != "hello world" {
+ t.Fatalf("expected content string 'hello world', got %T (%+v)", out["content"], out["content"])
+ }
+ if runtime.IsDeferredStream(out["content"]) {
+ t.Fatal("expected non-deferred output")
+ }
+}
+
+func TestGenerateStub_Invoke_DeferredStream(t *testing.T) {
+ stub := &stubStreamingInvoker{
+ deltas: []string{"gen ", "output"},
+ resp: &ChatInvokeResponse{Content: "gen output", Model: "m"},
+ }
+ withStubInvoker(t, stub)
+ ctx := runtime.WithComponentExecutionOptions(t.Context(), runtime.ComponentExecutionOptions{
+ DeferLLMToMessage: true,
+ })
+
+ g, err := NewGenerateStub(map[string]any{"prompt": "hi", "llm_id": "m"})
+ if err != nil {
+ t.Fatalf("NewGenerateStub: %v", err)
+ }
+ out, err := g.Invoke(ctx, nil, nil)
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ deferred, ok := out["content"].(*runtime.DeferredStream)
+ if !ok || deferred == nil {
+ t.Fatalf("expected *runtime.DeferredStream, got %T (%+v)", out["content"], out["content"])
+ }
+
+ var contentDeltas []string
+ sink := func(contentDelta, _ string) {
+ if contentDelta != "" {
+ contentDeltas = append(contentDeltas, contentDelta)
+ }
+ }
+ result, err := deferred.Open(t.Context(), sink)
+ if err != nil {
+ t.Fatalf("Open: %v", err)
+ }
+ if len(contentDeltas) != 2 || contentDeltas[0] != "gen " || contentDeltas[1] != "output" {
+ t.Errorf("content deltas = %+v, want ['gen ', 'output']", contentDeltas)
+ }
+ if result["content"] != "gen output" {
+ t.Errorf("result content = %v, want gen output", result["content"])
+ }
+}
diff --git a/internal/agent/runtime/context.go b/internal/agent/runtime/context.go
index 97e69ef3d8..f8a7837d9f 100644
--- a/internal/agent/runtime/context.go
+++ b/internal/agent/runtime/context.go
@@ -85,13 +85,14 @@ type deferredNodeRegistryCtxKey struct{}
type deferredNodeRegistry struct {
mu sync.Mutex
- completions map[string]func()
+ completions map[string]func(err error)
}
// ComponentExecutionOptions carries compile-time graph decisions into a
// component invocation without leaking internal flags into DSL parameters.
type ComponentExecutionOptions struct {
DeferAgentToMessage bool
+ DeferLLMToMessage bool
SuppressAgentMessageEvents bool
}
@@ -109,11 +110,11 @@ func ComponentExecutionOptionsFromContext(ctx context.Context) ComponentExecutio
// lazy stream.
func WithDeferredNodeRegistry(ctx context.Context) context.Context {
return context.WithValue(ctx, deferredNodeRegistryCtxKey{}, &deferredNodeRegistry{
- completions: make(map[string]func()),
+ completions: make(map[string]func(err error)),
})
}
-func RegisterDeferredNode(ctx context.Context, nodeID string, complete func()) {
+func RegisterDeferredNode(ctx context.Context, nodeID string, complete func(err error)) {
registry, _ := ctx.Value(deferredNodeRegistryCtxKey{}).(*deferredNodeRegistry)
if registry == nil || nodeID == "" || complete == nil {
return
@@ -123,7 +124,7 @@ func RegisterDeferredNode(ctx context.Context, nodeID string, complete func()) {
registry.mu.Unlock()
}
-func CompleteDeferredNode(ctx context.Context, nodeID string) {
+func CompleteDeferredNode(ctx context.Context, nodeID string, err error) {
registry, _ := ctx.Value(deferredNodeRegistryCtxKey{}).(*deferredNodeRegistry)
if registry == nil {
return
@@ -133,17 +134,21 @@ func CompleteDeferredNode(ctx context.Context, nodeID string) {
delete(registry.completions, nodeID)
registry.mu.Unlock()
if complete != nil {
- complete()
+ complete(err)
}
}
-func CompleteAllDeferredNodes(ctx context.Context) {
+func CompleteAllDeferredNodes(ctx context.Context, errs ...error) {
registry, _ := ctx.Value(deferredNodeRegistryCtxKey{}).(*deferredNodeRegistry)
if registry == nil {
return
}
+ var err error
+ if len(errs) > 0 {
+ err = errs[0]
+ }
registry.mu.Lock()
- callbacks := make([]func(), 0, len(registry.completions))
+ callbacks := make([]func(error), 0, len(registry.completions))
for nodeID, complete := range registry.completions {
callbacks = append(callbacks, complete)
delete(registry.completions, nodeID)
@@ -151,7 +156,7 @@ func CompleteAllDeferredNodes(ctx context.Context) {
registry.mu.Unlock()
for _, complete := range callbacks {
if complete != nil {
- complete()
+ complete(err)
}
}
}