RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(task): sink parser page-cap override into pipeline package (#17905)
Jackbb96bb687dcd
diff --git a/internal/ingestion/pipeline/parser_page_cap.go b/internal/ingestion/pipeline/parser_page_cap.gonew file mode 100644index 0000000000..94cab3eef1--- /dev/null+++ b/internal/ingestion/pipeline/parser_page_cap.go@@ -0,0 +1,109 @@+package pipeline++import (+ "encoding/json"+ "fmt"+)++// UnwrapCanvasDSL decodes a raw pipeline DSL and strips the optional canvas+// envelope {"dsl": {...}}, returning the inner components-carrying map. A raw+// (non-enveloped) DSL is returned unchanged. It is the []byte entry point used+// by helpers that need the inner DSL (ExtractParserCpnID,+// BuildParserPageCapOverride) so envelope-handling lives in exactly one place.+func UnwrapCanvasDSL(raw []byte) (map[string]any, error) {+ var top map[string]any+ if err := json.Unmarshal(raw, &top); err != nil {+ return nil, fmt.Errorf("UnwrapCanvasDSL: decode: %w", err)+ }+ if top == nil {+ return nil, errNilDSL+ }+ if env, ok := top["dsl"].(map[string]any); ok && len(env) > 0 {+ return env, nil+ }+ return top, nil+}++// ExtractParserCpnID returns the cpnID of the first component whose+// component_name equals parserComponentName, or "" when no such component+// exists or the DSL cannot be read. dsl may be enveloped; it is unwrapped+// first.+func ExtractParserCpnID(dsl []byte, parserComponentName string) string {+ inner, err := UnwrapCanvasDSL(dsl)+ if err != nil {+ return ""+ }+ innerJSON, err := json.Marshal(inner)+ if err != nil {+ return ""+ }+ schemas, err := ExtractAllComponentParams(innerJSON)+ if err != nil {+ return ""+ }+ for _, s := range schemas {+ if s.ComponentName == parserComponentName {+ return s.CpnID+ }+ }+ return ""+}++// BuildParserPageCapOverride returns parserConfig with a page cap injected for+// the Parser component, keyed by its cpnID and the document's filetype family.+//+// It is debug-agnostic: capPages and familyOf are supplied by the caller, so+// the function carries no debug-specific semantics and is reusable for any+// page-cap scenario. docType is the uploaded file extension (e.g. "pdf");+// familyOf maps it to a parser setup family (callers pass+// component.ParserFileFamily); parserComponentName is the Parser component+// name (callers pass component.ComponentNameParser) — kept as a parameter so+// the pipeline package does not import component.+//+// When no Parser component is found or the family is empty, the call is a+// no-op and parserConfig is returned unchanged. An explicit "pages" cap+// already present under cpnID+family is respected and left untouched — the+// cap is only a fallback. The injected shape is []any{[]any{1, capPages}}: a+// JSON-decoded list of 1-indexed inclusive ranges, the exact form the+// deepdoc/pdf parser's NormalizePDFPages consumes via+// ParserConfig[cpnID][family]["pages"].+func BuildParserPageCapOverride(+ parserConfig map[string]any,+ dsl []byte,+ docType string,+ capPages int,+ parserComponentName string,+ familyOf func(string) string,+) map[string]any {+ if parserConfig == nil {+ parserConfig = map[string]any{}+ }+ parserCpnID := ExtractParserCpnID(dsl, parserComponentName)+ if parserCpnID == "" {+ return parserConfig+ }+ family := familyOf(docType)+ if family == "" {+ return parserConfig+ }+ // Respect an explicit page cap already present under cpnID + family.+ if cpnEntry, ok := parserConfig[parserCpnID].(map[string]any); ok {+ if famEntry, ok := cpnEntry[family].(map[string]any); ok {+ if _, has := famEntry["pages"]; has {+ return parserConfig+ }+ }+ }+ cpnEntry, ok := parserConfig[parserCpnID].(map[string]any)+ if !ok {+ cpnEntry = map[string]any{}+ parserConfig[parserCpnID] = cpnEntry+ }+ famEntry, ok := cpnEntry[family].(map[string]any)+ if !ok {+ famEntry = map[string]any{}+ cpnEntry[family] = famEntry+ }+ famEntry["pages"] = []any{[]any{1, capPages}}+ return parserConfig+}diff --git a/internal/ingestion/pipeline/parser_page_cap_test.go b/internal/ingestion/pipeline/parser_page_cap_test.gonew file mode 100644index 0000000000..495fb89cc6--- /dev/null+++ b/internal/ingestion/pipeline/parser_page_cap_test.go@@ -0,0 +1,120 @@+package pipeline++import (+ "reflect"+ "testing"+)++// testParserComponentName mirrors component.ComponentNameParser ("Parser").+// The pipeline package deliberately does NOT import component (no reverse+// dependency); callers inject the name, so the test uses the literal here.+const testParserComponentName = "Parser"++// envelopedDSL wraps a components map in the canvas envelope {"dsl": {...}}.+func envelopedDSL(components string) []byte {+ return []byte(`{"dsl": {"components": ` + components + `}}`)+}++// TestExtractParserCpnID verifies the Parser cpnID is discovered from both+// enveloped and raw DSL, and "" is returned when no Parser component exists.+func TestExtractParserCpnID(t *testing.T) {+ // enveloped DSL with a Parser component.+ dsl := envelopedDSL(`{"Parser:Abc": {"obj": {"component_name": "Parser", "params": {}}}}`)+ if got := ExtractParserCpnID(dsl, testParserComponentName); got != "Parser:Abc" {+ t.Fatalf("enveloped: want Parser:Abc, got %q", got)+ }++ // no Parser component -> "".+ dslNo := envelopedDSL(`{"Tokenizer:X": {"obj": {"component_name": "Tokenizer", "params": {}}}}`)+ if got := ExtractParserCpnID(dslNo, testParserComponentName); got != "" {+ t.Fatalf("no parser: want empty, got %q", got)+ }++ // raw (non-enveloped) inner DSL is also accepted.+ raw := []byte(`{"components": {"Parser:Z": {"obj": {"component_name": "Parser", "params": {}}}}}`)+ if got := ExtractParserCpnID(raw, testParserComponentName); got != "Parser:Z" {+ t.Fatalf("raw: want Parser:Z, got %q", got)+ }+}++// TestBuildParserPageCapOverride verifies the debug-agnostic page-cap override:+// normal injection, respect-existing-cap, unknown-family no-op, no-parser no-op.+func TestBuildParserPageCapOverride(t *testing.T) {+ familyOf := func(ext string) string {+ if ext == "pdf" {+ return "pdf"+ }+ return ""+ }+ const docType = "pdf"+ dsl := envelopedDSL(`{"Parser:Abc": {"obj": {"component_name": "Parser", "params": {}}}}`)++ // 1. normal: enveloped DSL + pdf -> pages cap injected.+ pc := map[string]any{}+ out := BuildParserPageCapOverride(pc, dsl, docType, 2, testParserComponentName, familyOf)+ fam, ok := out["Parser:Abc"].(map[string]any)["pdf"].(map[string]any)+ if !ok {+ t.Fatalf("missing cpnID/family entry: %#v", out)+ }+ if !reflect.DeepEqual(fam["pages"], []any{[]any{1, 2}}) {+ t.Fatalf("pages shape wrong: %#v", fam["pages"])+ }++ // 2. respect an explicit existing cap (fallback, not overwrite).+ pc2 := map[string]any{+ "Parser:Abc": map[string]any{+ "pdf": map[string]any{"pages": []any{[]any{1, 99}}},+ },+ }+ out2 := BuildParserPageCapOverride(pc2, dsl, docType, 2, testParserComponentName, familyOf)+ fam2 := out2["Parser:Abc"].(map[string]any)["pdf"].(map[string]any)+ if !reflect.DeepEqual(fam2["pages"], []any{[]any{1, 99}}) {+ t.Fatalf("existing cap must be respected, got %#v", fam2["pages"])+ }++ // 3. unknown docType -> empty family -> no-op.+ out3 := BuildParserPageCapOverride(map[string]any{}, dsl, "xyz", 2, testParserComponentName, familyOf)+ if len(out3) != 0 {+ t.Fatalf("unknown docType should be no-op, got %#v", out3)+ }++ // 4. no Parser component -> no-op.+ dslNoParser := envelopedDSL(`{"Tokenizer:X": {"obj": {"component_name": "Tokenizer", "params": {}}}}`)+ out4 := BuildParserPageCapOverride(map[string]any{}, dslNoParser, docType, 2, testParserComponentName, familyOf)+ if len(out4) != 0 {+ t.Fatalf("no Parser should be no-op, got %#v", out4)+ }+}++// TestUnwrapCanvasDSL verifies the envelope is stripped and a raw DSL passes+// through unchanged; an empty/invalid payload errors.+func TestUnwrapCanvasDSL(t *testing.T) {+ env := []byte(`{"dsl": {"components": {}}}`)+ inner, err := UnwrapCanvasDSL(env)+ if err != nil {+ t.Fatalf("unexpected error: %v", err)+ }+ if _, ok := inner["components"]; !ok {+ t.Fatalf("envelope not stripped: %#v", inner)+ }++ raw := []byte(`{"components": {}}`)+ if _, err := UnwrapCanvasDSL(raw); err != nil {+ t.Fatalf("raw DSL should pass through, got error: %v", err)+ }++ // an empty-but-valid object passes through (matches UnwrapCanvasDSL: only+ // a truly nil/unparseable DSL errors).+ empty, err := UnwrapCanvasDSL([]byte(`{}`))+ if err != nil {+ t.Fatalf("empty object should not error, got: %v", err)+ }+ if len(empty) != 0 {+ t.Fatalf("empty object should yield empty map, got: %#v", empty)+ }++ // invalid JSON errors.+ if _, err := UnwrapCanvasDSL([]byte(`not json`)); err == nil {+ t.Fatalf("invalid JSON should error")+ }+}diff --git a/internal/ingestion/pipeline/pipeline.go b/internal/ingestion/pipeline/pipeline.goindex 9f8abfb804..e106d81901 100644--- a/internal/ingestion/pipeline/pipeline.go+++ b/internal/ingestion/pipeline/pipeline.go@@ -140,11 +140,10 @@ func WithProgressSink(s ProgressSink) PipelineOption {// It accepts either the inner canvas DSL or the template wrapper whose// top-level `dsl` field carries that canvas.func NewPipelineFromDSL(dsl []byte, taskID string, opts ...PipelineOption) (*Pipeline, error) {- var raw map[string]any- if err := json.Unmarshal(dsl, &raw); err != nil {- return nil, fmt.Errorf("pipeline: decode DSL: %w", err)- }- canvasDSL, err := unwrapCanvasDSL(raw)+ // UnwrapCanvasDSL is the single source of truth for stripping the+ // optional {"dsl": {...}} canvas envelope; it also reports a nil/unparseable+ // DSL.+ canvasDSL, err := UnwrapCanvasDSL(dsl)if err != nil {return nil, err}@@ -181,20 +180,6 @@ func (p *Pipeline) WithComponentFactory(factory runtime.ComponentFactory) *Pipelreturn p}-func unwrapCanvasDSL(raw map[string]any) (map[string]any, error) {- if len(raw) == 0 {- return nil, errNilDSL- }- if rawDSL, ok := raw["dsl"]; ok {- canvasDSL, ok := rawDSL.(map[string]any)- if !ok || len(canvasDSL) == 0 {- return nil, errNilDSL- }- return canvasDSL, nil- }- return raw, nil-}-func mergeInto(dst, src map[string]any) map[string]any {if src == nil {return dstdiff --git a/internal/ingestion/task/debug_pages_integration_test.go b/internal/ingestion/task/debug_pages_integration_test.goindex 6d38d2dbc2..d4c2694893 100644--- a/internal/ingestion/task/debug_pages_integration_test.go+++ b/internal/ingestion/task/debug_pages_integration_test.go@@ -27,6 +27,9 @@ import ("testing""time"+ "go.uber.org/zap"+ "go.uber.org/zap/zaptest/observer"+ "ragflow/internal/common""ragflow/internal/dao""ragflow/internal/entity""ragflow/internal/ingestion/component"@@ -42,9 +45,9 @@ import (// real pdfium parser, in-memory sqlite + storage)://// - Uncapped baseline: an explicit cpnID+family page cap of [[1, 1000000]]-// is supplied in ParserConfig. injectDebugPageCap must RESPECT it, so the+// is supplied in ParserConfig. pipeline.BuildParserPageCapOverride must RESPECT it, so the// parser reads every page.-// - Capped: no page cap is supplied, so injectDebugPageCap injects the+// - Capped: no page cap is supplied, so pipeline.BuildParserPageCapOverride injects the// debug default [[1, debugPageCapPages]], and the parser reads only the// leading pages.//@@ -79,7 +82,7 @@ func TestExecute_DebugViaEntry_HonorsPagesCap_Integration(t *testing.T) {// envelopeDSL is stored verbatim on the canvas (this is what production// persists), so loadDSLFromCanvas marshals the envelope and Run receives- // it. injectDebugPageCap must unwrap it to find the Parser cpnID.+ // it. pipeline.BuildParserPageCapOverride must unwrap it to find the Parser cpnID.var envelope struct {DSL json.RawMessage `json:"dsl"`}@@ -129,7 +132,7 @@ func TestExecute_DebugViaEntry_HonorsPagesCap_Integration(t *testing.T) {t.Cleanup(func() { _ = realDB.Where("id = ?", canvasID).Delete(&entity.UserCanvas{}).Error })// Explicit "parse all pages" cap (JSON-decoded []any form, the shape the- // parser actually consumes). injectDebugPageCap must respect it and leave+ // parser actually consumes). pipeline.BuildParserPageCapOverride must respect it and leave// it untouched, so the parser reads every page of the PDF.allPages := map[string]any{parserCpnID: map[string]any{@@ -142,7 +145,7 @@ func TestExecute_DebugViaEntry_HonorsPagesCap_Integration(t *testing.T) {newDebugCtx := func(parserConfig map[string]any) *TaskContext {// A canvas-debug (dry-run) context carries no KB: KB.ID == "" is the// single debug signal. The executor then skips the persist stage and- // injects the debug page cap (see injectDebugPageCap, gated on+ // injects the debug page cap (see pipeline.BuildParserPageCapOverride, gated on// KB.ID == "").return &TaskContext{Doc: entity.Document{@@ -163,6 +166,15 @@ func TestExecute_DebugViaEntry_HonorsPagesCap_Integration(t *testing.T) {}}+ // Capture warnings during the runs so we assert the envelope-unwrap fix+ // did not regress into noisy warnings: with a well-formed template the+ // parserConfig cpnID matches the DSL, so warnUnknownComponentParams must+ // NOT emit its "not present in the pipeline DSL" warning.+ core, recorded := observer.New(zap.NewAtomicLevelAt(zap.DebugLevel))+ oldLogger := common.Logger+ common.Logger = zap.New(core)+ defer func() { common.Logger = oldLogger }()+// Uncapped baseline: explicit pages=all → executor respects override.uncappedExec, err := NewPipelineExecutor(newDebugCtx(allPages), canvasID, 0)if err != nil {@@ -183,6 +195,15 @@ func TestExecute_DebugViaEntry_HonorsPagesCap_Integration(t *testing.T) {t.Fatalf("Execute (capped): %v", err)}+ // A well-formed template's parserConfig cpnID matches the DSL, so the+ // unknown-cpnID guard must stay silent. If it warned here, the envelope+ // was not unwrapped (the pre-fix no-op regression).+ for _, e := range recorded.All() {+ if strings.Contains(e.Message, "not present in the pipeline DSL") {+ t.Errorf("warnUnknownComponentParams emitted an unknown-cpnID warning during a well-formed debug run (envelope-unwrap regression?): %s", e.Message)+ }+ }+uncappedLen := len(joinedChunks(uncapped.Chunks))cappedLen := len(joinedChunks(capped.Chunks))if uncappedLen == 0 {diff --git a/internal/ingestion/task/debug_result_dsl.go b/internal/ingestion/task/debug_result_dsl.goindex e9131e65d8..c6ebbda85e 100644--- a/internal/ingestion/task/debug_result_dsl.go+++ b/internal/ingestion/task/debug_result_dsl.go@@ -30,6 +30,8 @@ import ("encoding/json""fmt""strings"++ pipelinepkg "ragflow/internal/ingestion/pipeline")// ResultSink is an OPTIONAL capability a ProgressSink may implement to receive@@ -80,9 +82,11 @@ func BuildDebugResultDSL(dsl string, output map[string]any) (map[string]any, errif err := json.Unmarshal([]byte(dsl), &tpl); err != nil {return nil, fmt.Errorf("BuildDebugResultDSL: unmarshal dsl: %w", err)}+ // Unwrap the canvas envelope via the shared helper so envelope handling+ // lives in exactly one place (pipeline.UnwrapCanvasDSL).root := tpl- if nested, ok := tpl["dsl"].(map[string]any); ok {- root = nested+ if inner, err := pipelinepkg.UnwrapCanvasDSL([]byte(dsl)); err == nil && inner != nil {+ root = inner}components, ok := root["components"].(map[string]any)diff --git a/internal/ingestion/task/debug_test.go b/internal/ingestion/task/debug_test.goindex d39681f097..ed028d68d0 100644--- a/internal/ingestion/task/debug_test.go+++ b/internal/ingestion/task/debug_test.go@@ -23,9 +23,13 @@ import ("os""path/filepath""reflect"+ "strings""testing"+ "go.uber.org/zap"+ "go.uber.org/zap/zaptest/observer""gorm.io/gorm"+ "ragflow/internal/common""ragflow/internal/entity""ragflow/internal/ingestion/component""ragflow/internal/ingestion/pipeline"@@ -37,8 +41,8 @@ import (// the debug signal used across the pipeline). The parser page cap is no longer// stored as a flat ParserConfig key here — flat keys are dropped by the// override_params merge and never reach the parser. It is injected at run time-// by injectDebugPageCap via Run's override_params channel (see-// TestInjectDebugPageCap).+// by pipeline.BuildParserPageCapOverride via Run's override_params channel+// (see TestInjectDebugPageCap).func TestNewDebugTaskContext_InjectsDebugID(t *testing.T) {taskCtx := NewDebugTaskContext("t1", "canvas-1", "doc.pdf", []byte("page one\fpage two\fpage three"))@@ -127,6 +131,10 @@ func TestExecute_DebugViaEntry(t *testing.T) {// storage JSON round-trip and the shape the deepdoc pdf parser consumes// (NormalizePDFPages requires []any, not a Go [][]int).//+// The DSL/parser-family knowledge now lives in pipeline.BuildParserPageCapOverride+// (debug-agnostic); this test drives that helper directly to pin the behavior+// the executor relies on.+//// It also pins the regression: a flat top-level "pages" key would be dropped// by the override_params merge and never reach the parser, so the cap must be// nested under the cpnID.@@ -159,9 +167,15 @@ func TestInjectDebugPageCap(t *testing.T) {t.Fatal("template has no Parser component")}+ apply := func(cfg map[string]any, dslArg string, docType string) map[string]any {+ return pipeline.BuildParserPageCapOverride(+ cfg, []byte(dslArg), docType, debugPageCapPages,+ component.ComponentNameParser, component.ParserFileFamily)+ }+t.Run("pdf injects [1,2] under cpnID+family", func(t *testing.T) {parserConfig := map[string]any{}- injectDebugPageCap(dsl, parserConfig, "pdf")+ apply(parserConfig, dsl, "pdf")famEntry, ok := parserConfig[parserCpnID].(map[string]any)if !ok {t.Fatalf("parserConfig[%q] = %T, want map[string]any", parserCpnID, parserConfig[parserCpnID])@@ -177,7 +191,7 @@ func TestInjectDebugPageCap(t *testing.T) {t.Run("docx injects under docx family", func(t *testing.T) {parserConfig := map[string]any{}- injectDebugPageCap(dsl, parserConfig, "docx")+ apply(parserConfig, dsl, "docx")famEntry, ok := parserConfig[parserCpnID].(map[string]any)if !ok {t.Fatalf("parserConfig[%q] = %T, want map[string]any", parserCpnID, parserConfig[parserCpnID])@@ -193,7 +207,7 @@ func TestInjectDebugPageCap(t *testing.T) {t.Run("empty docType is a no-op", func(t *testing.T) {parserConfig := map[string]any{}- injectDebugPageCap(dsl, parserConfig, "")+ apply(parserConfig, dsl, "")if len(parserConfig) != 0 {t.Errorf("parserConfig = %v, want empty (no family derivable from empty docType)", parserConfig)}@@ -205,7 +219,7 @@ func TestInjectDebugPageCap(t *testing.T) {"pdf": map[string]any{"parse_method": "deepdoc"},},}- injectDebugPageCap(dsl, parserConfig, "pdf")+ apply(parserConfig, dsl, "pdf")famEntry := parserConfig[parserCpnID].(map[string]any)pdf := famEntry["pdf"].(map[string]any)if pdf["parse_method"] != "deepdoc" {@@ -218,11 +232,11 @@ func TestInjectDebugPageCap(t *testing.T) {t.Run("envelope dsl form (production shape)", func(t *testing.T) {// In production dsl is the canvas envelope {"dsl": {"components": ...}},- // not the bare components map. injectDebugPageCap must still find the- // Parser cpnID after unwrapping.+ // not the bare components map. The helper must still find the Parser+ // cpnID after unwrapping.wrapped := fmt.Sprintf(`{"dsl":%s}`, string(envelope.DSL))parserConfig := map[string]any{}- injectDebugPageCap(wrapped, parserConfig, "pdf")+ apply(parserConfig, wrapped, "pdf")famEntry, ok := parserConfig[parserCpnID].(map[string]any)if !ok {t.Fatalf("parserConfig[%q] = %T, want map[string]any (envelope dsl must unwrap)", parserCpnID, parserConfig[parserCpnID])@@ -244,7 +258,7 @@ func TestInjectDebugPageCap(t *testing.T) {"pdf": map[string]any{"pages": []any{[]any{1, 1000000}}},},}- injectDebugPageCap(dsl, parserConfig, "pdf")+ apply(parserConfig, dsl, "pdf")famEntry := parserConfig[parserCpnID].(map[string]any)pdf := famEntry["pdf"].(map[string]any)if !reflect.DeepEqual(pdf["pages"], []any{[]any{1, 1000000}}) {@@ -252,3 +266,82 @@ func TestInjectDebugPageCap(t *testing.T) {}})}++// TestWarnUnknownComponentParamsDetectsUnknownCPNFromEnvelope pins the fix for+// the enveloped-DSL no-op bug: warnUnknownComponentParams previously passed the+// raw (enveloped) DSL straight to ExtractAllComponentParams, whose "components"+// key is nested under "dsl", so it errored and silently returned — never+// detecting unknown cpnIDs in production. The helper now unwraps the envelope+// first, so an unknown cpnID in parserConfig is actually surfaced.+func TestWarnUnknownComponentParamsDetectsUnknownCPNFromEnvelope(t *testing.T) {+ core, recorded := observer.New(zap.NewAtomicLevelAt(zap.DebugLevel))+ old := common.Logger+ common.Logger = zap.New(core)+ defer func() { common.Logger = old }()++ // Enveloped DSL (production shape) carrying only a Parser component.+ dsl := `{"dsl": {"components": {"Parser:Abc": {"obj": {"component_name": "Parser", "params": {}}}}}}`+ // parserConfig references a cpnID NOT present in the DSL -> must be warned.+ parserConfig := map[string]any{+ "Parser:Unknown": map[string]any{"pdf": map[string]any{}},+ }++ warnUnknownComponentParams(dsl, parserConfig)++ found := false+ for _, e := range recorded.All() {+ if strings.Contains(e.Message, "Parser:Unknown") {+ found = true+ }+ }+ if !found {+ t.Fatalf("expected a warning about unknown cpnID Parser:Unknown (envelope DSL must be unwrapped); got logs: %v", recorded.All())+ }+}++// TestBuildDebugResultDSL_Envelope pins that BuildDebugResultDSL unwraps the+// canvas envelope before reading "components" — the same shared+// pipeline.UnwrapCanvasDSL the cap override and warnUnknownComponentParams+// use. An enveloped DSL (production shape {"dsl": {...}}) must resolve the+// components map exactly like the equivalent raw (non-enveloped) DSL.+func TestBuildDebugResultDSL_Envelope(t *testing.T) {+ const compID = "Parser:Abc"+ rawDSL := `{"components": {"` + compID + `": {"obj": {"component_name": "Parser", "params": {"parse_method": "deepdoc"}}}}}`+ output := map[string]any{+ "state": map[string]any{+ compID: map[string]any{"chunks": []any{map[string]any{"text": "hi"}}},+ },+ }++ // Raw (non-enveloped) DSL.+ rawRes, err := BuildDebugResultDSL(rawDSL, output)+ if err != nil {+ t.Fatalf("raw DSL: %v", err)+ }+ rawComps, ok := rawRes["components"].(map[string]any)+ if !ok {+ t.Fatalf("raw DSL: components missing: %#v", rawRes)+ }+ if _, ok := rawComps[compID]; !ok {+ t.Fatalf("raw DSL: components missing %q: %#v", compID, rawComps)+ }++ // Enveloped DSL (production shape) must unwrap to the same result.+ envDSL := `{"dsl": ` + rawDSL + `}`+ envRes, err := BuildDebugResultDSL(envDSL, output)+ if err != nil {+ t.Fatalf("enveloped DSL: %v", err)+ }+ envComps, ok := envRes["components"].(map[string]any)+ if !ok {+ t.Fatalf("enveloped DSL: components missing (envelope not unwrapped?): %#v", envRes)+ }+ if _, ok := envComps[compID]; !ok {+ t.Fatalf("enveloped DSL: components missing %q (envelope not unwrapped?): %#v", compID, envComps)+ }++ // The two shapes must yield an identical component output.+ if !reflect.DeepEqual(rawComps[compID], envComps[compID]) {+ t.Fatalf("enveloped and raw DSL produced different results:\nraw=%#v\nenv=%#v", rawComps[compID], envComps[compID])+ }+}diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.goindex 8316db14f4..c0cbfbc994 100644--- a/internal/ingestion/task/pipeline_executor.go+++ b/internal/ingestion/task/pipeline_executor.go@@ -374,7 +374,22 @@ func warnUnknownComponentParams(dsl string, parserConfig map[string]any) {if len(parserConfig) == 0 {return}- schemas, err := pipelinepkg.ExtractAllComponentParams([]byte(dsl))+ // dsl arrives as the canvas ENVELOPE ({ "dsl": { "components": ... } }) in+ // production, so it must be unwrapped before ExtractAllComponentParams+ // runs (that helper expects the inner DSL). The previous direct call+ // passed the enveloped DSL, whose "components" key is nested under "dsl",+ // so it silently returned an error and made this guard a no-op.+ inner, err := pipelinepkg.UnwrapCanvasDSL([]byte(dsl))+ if err != nil {+ common.Warn(fmt.Sprintf("warnUnknownComponentParams: cannot parse DSL to validate component params: %v", err))+ return+ }+ innerJSON, err := json.Marshal(inner)+ if err != nil {+ common.Warn(fmt.Sprintf("warnUnknownComponentParams: cannot re-encode DSL: %v", err))+ return+ }+ schemas, err := pipelinepkg.ExtractAllComponentParams(innerJSON)if err != nil {common.Warn(fmt.Sprintf("warnUnknownComponentParams: cannot parse DSL to validate component params: %v", err))return@@ -472,9 +487,12 @@ func (s *PipelineExecutor) runPipelineWithDSL(ctx context.Context, dsl string) (// and the document's filetype family. It is NOT passed through pipeline// inputs: the parser selects pages from ParserConfig[cpnID][family]// ["pages"] (a list of 1-indexed inclusive ranges), exactly mirroring- // NormalizeParserConfigPages / pdf_pages_test.go. See injectDebugPageCap.+ // NormalizeParserConfigPages / pdf_pages_test.go. The DSL/parser-family+ // knowledge now lives in pipeline.BuildParserPageCapOverride.if debug {- injectDebugPageCap(dsl, parserConfig, s.taskCtx.Doc.Type)+ parserConfig = pipelinepkg.BuildParserPageCapOverride(+ parserConfig, []byte(dsl), s.taskCtx.Doc.Type,+ debugPageCapPages, component.ComponentNameParser, component.ParserFileFamily)}// Component params from Doc.ParserConfig — including the tenant LLM id@@ -511,79 +529,3 @@ func (s *PipelineExecutor) runPipelineWithDSL(ctx context.Context, dsl string) (// inclusive range [1, debugPageCapPages], matching the production// ParserConfig[cpnID][filetype]["pages"] shape (see NormalizeParserConfigPages).const debugPageCapPages = 2--// injectDebugPageCap wires the canvas-debug page cap into parserConfig using-// the SAME channel the production ParserConfig travels through: Run's-// override_params (the 3rd argument), keyed by the Parser component's cpnID-// and the document's filetype family. It must NOT be passed as a pipeline-// input — the parser selects pages from ParserConfig[cpnID][family]["pages"],-// which the deepdoc/pdf parser consumes as a list of page ranges.-//-// dsl is the raw pipeline DSL (used only to discover the Parser component's-// cpnID); docType is the uploaded file's extension (e.g. "pdf"). When no-// Parser component can be found, or the docType yields no known family, the-// call is a no-op (the run parses everything, which is safe).-//-// dsl arrives as the canvas ENVELOPE ({ "dsl": { "components": ... } }) in-// production (loadDSLFromCanvas marshals canvas.DSL), so it is unwrapped the-// same way NewPipelineFromDSL does before ExtractAllComponentParams runs.-// An explicit page cap already present in parserConfig (keyed by cpnID +-// family) is respected and left untouched — the debug default is only a-// fallback, so a debug run can honour a narrower or wider caller-supplied cap.-func injectDebugPageCap(dsl string, parserConfig map[string]any, docType string) {- // Unwrap the canvas envelope to the inner components map.- var raw map[string]any- if err := json.Unmarshal([]byte(dsl), &raw); err != nil {- return- }- if env, ok := raw["dsl"].(map[string]any); ok && len(env) > 0 {- raw = env- }- inner, err := json.Marshal(raw)- if err != nil {- return- }- schemas, err := pipelinepkg.ExtractAllComponentParams(inner)- if err != nil {- return- }- var parserCpnID string- for _, s := range schemas {- if s.ComponentName == component.ComponentNameParser {- parserCpnID = s.CpnID- break- }- }- if parserCpnID == "" {- return- }- family := component.ParserFileFamily(docType)- if family == "" {- return- }- // Respect an explicit page cap already present under cpnID + family.- if cpnEntry, ok := parserConfig[parserCpnID].(map[string]any); ok {- if famEntry, ok := cpnEntry[family].(map[string]any); ok {- if _, has := famEntry["pages"]; has {- return- }- }- }- cpnEntry, ok := parserConfig[parserCpnID].(map[string]any)- if !ok {- cpnEntry = map[string]any{}- parserConfig[parserCpnID] = cpnEntry- }- famEntry, ok := cpnEntry[family].(map[string]any)- if !ok {- famEntry = map[string]any{}- cpnEntry[family] = famEntry- }- // pages is delivered as a JSON-decoded map — the very shape a ParserConfig- // arrives in from the API/storage JSON round-trip: a []any of [from,to]- // pairs. The deepdoc/pdf parser's NormalizePDFPages requires this- // []any-of-[]any form (not a Go [][]int), so it is built explicitly here.- // The shallow override_params merge in applyOverrideParams preserves this- // shape all the way to ConfigureFromSetup, so the cap is honoured.- famEntry["pages"] = []any{[]any{1, debugPageCapPages}}-}