Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(pipeline): ignore disconnected nodes during terminal detection (#19976)

Sbaaoui Idrissf583b5375532
diff --git a/internal/ingestion/pipeline/payload.go b/internal/ingestion/pipeline/payload.go
index d9e8cd7371..6bd2d3b698 100644
--- a/internal/ingestion/pipeline/payload.go
+++ b/internal/ingestion/pipeline/payload.go
@@ -20,6 +20,7 @@ import (
"encoding/json"
"fmt"
"sort"
+ "strings"
)
// ExtractPayload extracts the terminal component's output from a pipeline run
@@ -41,14 +42,23 @@ func ExtractPayload(dsl string, out map[string]any) (map[string]any, error) {
return nil, fmt.Errorf("pipeline requires exactly 1 terminal, got %d: %v", len(terminalIDs), terminalIDs)
}
payload, ok := out[terminalIDs[0]].(map[string]any)
+ if !ok {
+ if state, stateOK := out["state"].(map[string]any); stateOK {
+ payload, ok = state[terminalIDs[0]].(map[string]any)
+ } else if state, stateOK := out["state"].(map[string]map[string]any); stateOK {
+ payload, ok = state[terminalIDs[0]]
+ }
+ }
if !ok {
return nil, fmt.Errorf("run output missing terminal payload %q", terminalIDs[0])
}
return payload, nil
}
-// TerminalComponentIDs walks a raw DSL JSON and returns the sorted ids of
-// components that have no downstream connections (terminals / sinks).
+// TerminalComponentIDs walks the execution graph from its entry component(s)
+// and returns the sorted ids of reachable components that have no downstream
+// connections (terminals / sinks). Disconnected canvas components are not part
+// of a run and therefore must not affect terminal selection.
func TerminalComponentIDs(raw []byte) ([]string, error) {
var tpl map[string]any
if err := json.Unmarshal(raw, &tpl); err != nil {
@@ -62,12 +72,50 @@ func TerminalComponentIDs(raw []byte) ([]string, error) {
if !ok {
return nil, fmt.Errorf("canvas dsl missing components map")
}
- terminals := make([]string, 0, len(components))
+ roots := make([]string, 0, 2)
for id, rawComp := range components {
comp, ok := rawComp.(map[string]any)
if !ok {
return nil, fmt.Errorf("component %q has invalid type %T", id, rawComp)
}
+ if componentName(comp) == "file" || componentName(comp) == "begin" {
+ roots = append(roots, id)
+ }
+ }
+ // Minimal/raw DSLs used by older callers may omit component metadata. Keep
+ // their historical sink scan rather than inventing an entry point.
+ if len(roots) == 0 {
+ roots = make([]string, 0, len(components))
+ for id := range components {
+ roots = append(roots, id)
+ }
+ }
+
+ reachable := make(map[string]bool, len(components))
+ queue := append([]string(nil), roots...)
+ for len(queue) > 0 {
+ id := queue[0]
+ queue = queue[1:]
+ if reachable[id] {
+ continue
+ }
+ if _, ok := components[id]; !ok {
+ continue
+ }
+ reachable[id] = true
+ comp := components[id].(map[string]any)
+ if downstream, ok := comp["downstream"].([]any); ok {
+ for _, rawChild := range downstream {
+ if child, ok := rawChild.(string); ok {
+ queue = append(queue, child)
+ }
+ }
+ }
+ }
+
+ terminals := make([]string, 0, len(reachable))
+ for id := range reachable {
+ comp := components[id].(map[string]any)
switch downstream := comp["downstream"].(type) {
case nil:
terminals = append(terminals, id)
@@ -82,3 +130,15 @@ func TerminalComponentIDs(raw []byte) ([]string, error) {
sort.Strings(terminals)
return terminals, nil
}
+
+func componentName(comp map[string]any) string {
+ if obj, ok := comp["obj"].(map[string]any); ok {
+ if name, ok := obj["component_name"].(string); ok {
+ return strings.ToLower(name)
+ }
+ }
+ if name, ok := comp["name"].(string); ok {
+ return strings.ToLower(name)
+ }
+ return ""
+}
diff --git a/internal/ingestion/pipeline/payload_test.go b/internal/ingestion/pipeline/payload_test.go
index 964ae94e72..8c7d626f07 100644
--- a/internal/ingestion/pipeline/payload_test.go
+++ b/internal/ingestion/pipeline/payload_test.go
@@ -49,6 +49,20 @@ func TestExtractPayload(t *testing.T) {
wantOK: true,
want: map[string]any{"text": "hi"},
},
+ {
+ name: "terminal payload in state snapshot",
+ dsl: `{"components":{"begin":{"downstream":["c1"]},"c1":{"downstream":null}}}`,
+ output: map[string]any{"state": map[string]any{"c1": map[string]any{"text": "from state"}}},
+ wantOK: true,
+ want: map[string]any{"text": "from state"},
+ },
+ {
+ name: "terminal payload in typed state snapshot",
+ dsl: `{"components":{"begin":{"downstream":["c1"]},"c1":{"downstream":null}}}`,
+ output: map[string]any{"state": map[string]map[string]any{"c1": {"text": "from typed state"}}},
+ wantOK: true,
+ want: map[string]any{"text": "from typed state"},
+ },
{
name: "zero terminals → error",
dsl: `{"components":{}}`,
@@ -130,27 +144,27 @@ func TestTerminalComponentIDs(t *testing.T) {
}{
{
name: "single terminal",
- raw: []byte(`{"components":{"c1":{"downstream":null}}}`),
+ raw: []byte(`{"components":{"begin":{"obj":{"component_name":"Begin"},"downstream":["c1"]},"c1":{"downstream":null}}}`),
want: []string{"c1"},
},
{
name: "multiple terminals sorted",
- raw: []byte(`{"components":{"c2":{"downstream":null},"c1":{"downstream":null}}}`),
+ raw: []byte(`{"components":{"begin":{"obj":{"component_name":"Begin"},"downstream":["c2","c1"]},"c2":{"downstream":null},"c1":{"downstream":null}}}`),
want: []string{"c1", "c2"},
},
{
name: "non-terminal with downstream connections excluded",
- raw: []byte(`{"components":{"c1":{"downstream":["c2"]},"c2":{"downstream":null}}}`),
+ raw: []byte(`{"components":{"begin":{"obj":{"component_name":"Begin"},"downstream":["c1"]},"c1":{"downstream":["c2"]},"c2":{"downstream":null}}}`),
want: []string{"c2"},
},
{
name: "empty downstream array counts as terminal",
- raw: []byte(`{"components":{"c1":{"downstream":[]}}}`),
+ raw: []byte(`{"components":{"begin":{"obj":{"component_name":"Begin"},"downstream":["c1"]},"c1":{"downstream":[]}}}`),
want: []string{"c1"},
},
{
name: "nested dsl template unwrapped",
- raw: []byte(`{"dsl":{"components":{"c1":{"downstream":null}}}}`),
+ raw: []byte(`{"dsl":{"components":{"begin":{"obj":{"component_name":"Begin"},"downstream":["c1"]},"c1":{"downstream":null}}}}`),
want: []string{"c1"},
},
{
@@ -159,10 +173,20 @@ func TestTerminalComponentIDs(t *testing.T) {
wantErr: "missing components map",
},
{
- name: "no terminals",
- raw: []byte(`{"components":{}}`),
+ name: "no terminals in reachable cycle",
+ raw: []byte(`{"components":{"begin":{"obj":{"component_name":"Begin"},"downstream":["a"]},"a":{"downstream":["b"]},"b":{"downstream":["a"]}}}`),
want: []string{},
},
+ {
+ name: "disconnected leaf is ignored",
+ raw: []byte(`{"components":{"begin":{"obj":{"component_name":"Begin"},"downstream":["connected"]},"connected":{"downstream":null},"TitleChunker:isolated":{"obj":{"component_name":"TitleChunker"},"downstream":null}}}`),
+ want: []string{"connected"},
+ },
+ {
+ name: "disconnected subgraph is ignored",
+ raw: []byte(`{"components":{"begin":{"obj":{"component_name":"Begin"},"downstream":["connected"]},"connected":{"downstream":null},"Parser:isolated":{"obj":{"component_name":"Parser"},"downstream":["Compiler:isolated"]},"Compiler:isolated":{"downstream":null}}}`),
+ want: []string{"connected"},
+ },
{
name: "invalid JSON",
raw: []byte(`{invalid`),