RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(pipeline): ignore disconnected nodes during terminal detection (#19976)
Sbaaoui Idrissf583b5375532
diff --git a/internal/ingestion/pipeline/payload.go b/internal/ingestion/pipeline/payload.goindex d9e8cd7371..6bd2d3b698 100644--- a/internal/ingestion/pipeline/payload.go+++ b/internal/ingestion/pipeline/payload.go@@ -20,6 +20,7 @@ import ("encoding/json""fmt""sort"+ "strings")// ExtractPayload extracts the terminal component's output from a pipeline run@@ -41,14 +42,23 @@ func ExtractPayload(dsl string, out map[string]any) (map[string]any, error) {return nil, fmt.Errorf("pipeline requires exactly 1 terminal, got %d: %v", len(terminalIDs), terminalIDs)}payload, ok := out[terminalIDs[0]].(map[string]any)+ if !ok {+ if state, stateOK := out["state"].(map[string]any); stateOK {+ payload, ok = state[terminalIDs[0]].(map[string]any)+ } else if state, stateOK := out["state"].(map[string]map[string]any); stateOK {+ payload, ok = state[terminalIDs[0]]+ }+ }if !ok {return nil, fmt.Errorf("run output missing terminal payload %q", terminalIDs[0])}return payload, nil}-// TerminalComponentIDs walks a raw DSL JSON and returns the sorted ids of-// components that have no downstream connections (terminals / sinks).+// TerminalComponentIDs walks the execution graph from its entry component(s)+// and returns the sorted ids of reachable components that have no downstream+// connections (terminals / sinks). Disconnected canvas components are not part+// of a run and therefore must not affect terminal selection.func TerminalComponentIDs(raw []byte) ([]string, error) {var tpl map[string]anyif err := json.Unmarshal(raw, &tpl); err != nil {@@ -62,12 +72,50 @@ func TerminalComponentIDs(raw []byte) ([]string, error) {if !ok {return nil, fmt.Errorf("canvas dsl missing components map")}- terminals := make([]string, 0, len(components))+ roots := make([]string, 0, 2)for id, rawComp := range components {comp, ok := rawComp.(map[string]any)if !ok {return nil, fmt.Errorf("component %q has invalid type %T", id, rawComp)}+ if componentName(comp) == "file" || componentName(comp) == "begin" {+ roots = append(roots, id)+ }+ }+ // Minimal/raw DSLs used by older callers may omit component metadata. Keep+ // their historical sink scan rather than inventing an entry point.+ if len(roots) == 0 {+ roots = make([]string, 0, len(components))+ for id := range components {+ roots = append(roots, id)+ }+ }++ reachable := make(map[string]bool, len(components))+ queue := append([]string(nil), roots...)+ for len(queue) > 0 {+ id := queue[0]+ queue = queue[1:]+ if reachable[id] {+ continue+ }+ if _, ok := components[id]; !ok {+ continue+ }+ reachable[id] = true+ comp := components[id].(map[string]any)+ if downstream, ok := comp["downstream"].([]any); ok {+ for _, rawChild := range downstream {+ if child, ok := rawChild.(string); ok {+ queue = append(queue, child)+ }+ }+ }+ }++ terminals := make([]string, 0, len(reachable))+ for id := range reachable {+ comp := components[id].(map[string]any)switch downstream := comp["downstream"].(type) {case nil:terminals = append(terminals, id)@@ -82,3 +130,15 @@ func TerminalComponentIDs(raw []byte) ([]string, error) {sort.Strings(terminals)return terminals, nil}++func componentName(comp map[string]any) string {+ if obj, ok := comp["obj"].(map[string]any); ok {+ if name, ok := obj["component_name"].(string); ok {+ return strings.ToLower(name)+ }+ }+ if name, ok := comp["name"].(string); ok {+ return strings.ToLower(name)+ }+ return ""+}diff --git a/internal/ingestion/pipeline/payload_test.go b/internal/ingestion/pipeline/payload_test.goindex 964ae94e72..8c7d626f07 100644--- a/internal/ingestion/pipeline/payload_test.go+++ b/internal/ingestion/pipeline/payload_test.go@@ -49,6 +49,20 @@ func TestExtractPayload(t *testing.T) {wantOK: true,want: map[string]any{"text": "hi"},},+ {+ name: "terminal payload in state snapshot",+ dsl: `{"components":{"begin":{"downstream":["c1"]},"c1":{"downstream":null}}}`,+ output: map[string]any{"state": map[string]any{"c1": map[string]any{"text": "from state"}}},+ wantOK: true,+ want: map[string]any{"text": "from state"},+ },+ {+ name: "terminal payload in typed state snapshot",+ dsl: `{"components":{"begin":{"downstream":["c1"]},"c1":{"downstream":null}}}`,+ output: map[string]any{"state": map[string]map[string]any{"c1": {"text": "from typed state"}}},+ wantOK: true,+ want: map[string]any{"text": "from typed state"},+ },{name: "zero terminals → error",dsl: `{"components":{}}`,@@ -130,27 +144,27 @@ func TestTerminalComponentIDs(t *testing.T) {}{{name: "single terminal",- raw: []byte(`{"components":{"c1":{"downstream":null}}}`),+ raw: []byte(`{"components":{"begin":{"obj":{"component_name":"Begin"},"downstream":["c1"]},"c1":{"downstream":null}}}`),want: []string{"c1"},},{name: "multiple terminals sorted",- raw: []byte(`{"components":{"c2":{"downstream":null},"c1":{"downstream":null}}}`),+ raw: []byte(`{"components":{"begin":{"obj":{"component_name":"Begin"},"downstream":["c2","c1"]},"c2":{"downstream":null},"c1":{"downstream":null}}}`),want: []string{"c1", "c2"},},{name: "non-terminal with downstream connections excluded",- raw: []byte(`{"components":{"c1":{"downstream":["c2"]},"c2":{"downstream":null}}}`),+ raw: []byte(`{"components":{"begin":{"obj":{"component_name":"Begin"},"downstream":["c1"]},"c1":{"downstream":["c2"]},"c2":{"downstream":null}}}`),want: []string{"c2"},},{name: "empty downstream array counts as terminal",- raw: []byte(`{"components":{"c1":{"downstream":[]}}}`),+ raw: []byte(`{"components":{"begin":{"obj":{"component_name":"Begin"},"downstream":["c1"]},"c1":{"downstream":[]}}}`),want: []string{"c1"},},{name: "nested dsl template unwrapped",- raw: []byte(`{"dsl":{"components":{"c1":{"downstream":null}}}}`),+ raw: []byte(`{"dsl":{"components":{"begin":{"obj":{"component_name":"Begin"},"downstream":["c1"]},"c1":{"downstream":null}}}}`),want: []string{"c1"},},{@@ -159,10 +173,20 @@ func TestTerminalComponentIDs(t *testing.T) {wantErr: "missing components map",},{- name: "no terminals",- raw: []byte(`{"components":{}}`),+ name: "no terminals in reachable cycle",+ raw: []byte(`{"components":{"begin":{"obj":{"component_name":"Begin"},"downstream":["a"]},"a":{"downstream":["b"]},"b":{"downstream":["a"]}}}`),want: []string{},},+ {+ name: "disconnected leaf is ignored",+ raw: []byte(`{"components":{"begin":{"obj":{"component_name":"Begin"},"downstream":["connected"]},"connected":{"downstream":null},"TitleChunker:isolated":{"obj":{"component_name":"TitleChunker"},"downstream":null}}}`),+ want: []string{"connected"},+ },+ {+ name: "disconnected subgraph is ignored",+ raw: []byte(`{"components":{"begin":{"obj":{"component_name":"Begin"},"downstream":["connected"]},"connected":{"downstream":null},"Parser:isolated":{"obj":{"component_name":"Parser"},"downstream":["Compiler:isolated"]},"Compiler:isolated":{"downstream":null}}}`),+ want: []string{"connected"},+ },{name: "invalid JSON",raw: []byte(`{invalid`),