RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(agent): save incomplete chunker delimiters and gate Run (#21371)
Sbaaoui Idrissba9bbffcd168
diff --git a/internal/agent/component/dynamic_params.go b/internal/agent/component/dynamic_params.goindex 7833486f2b..c77c28e860 100644--- a/internal/agent/component/dynamic_params.go+++ b/internal/agent/component/dynamic_params.go@@ -7,6 +7,7 @@ import ("strings""ragflow/internal/agent/tool"+ "ragflow/internal/ingestion/component/schema")// ErrCompilationTemplateRequired identifies a Compiler component without a@@ -20,7 +21,7 @@ func ValidateDynamicEntries(dsl map[string]any) error {// ValidateDraftDynamicEntries allows unfinished Message content, DataOperations// entries, Invoke mappings, Compiler configuration, input options, Loop settings,-// Iteration outputs, SQL connections, and search filters in saved drafts.+// Iteration outputs, SQL connections, search filters, and chunker delimiters in saved drafts.// All other checks remain unchanged.func ValidateDraftDynamicEntries(dsl map[string]any) error {return validateDynamicEntries(dsl, true)@@ -55,13 +56,8 @@ func validateDynamicParams(component string, params map[string]any, draft bool)if err := tool.ValidateSearchFilters(params, draft); err != nil {return fmt.Errorf("[%s] %w", component, err)}- // Delimiter entries are split tokens, not names: whitespace-only values- // ("\n", "\t") are valid delimiters — the chunker form's default row is a- // real newline — so only a zero-length string marks an incomplete row.- for _, field := range []string{"delimiters", "children_delimiters"} {- if values, ok := params[field].([]any); ok && containsEmpty(values) {- return fmt.Errorf("[%s] %s does not support empty entries", component, field)- }+ if err := schema.ValidateDelimiterEntries(params, draft); err != nil {+ return fmt.Errorf("[%s] %w", component, err)}switch strings.ToLower(component) {case "message":@@ -292,18 +288,6 @@ func containsBlank(values []any) bool {return false}-// containsEmpty reports whether any entry is a zero-length string. Unlike-// containsBlank it keeps whitespace-only entries, which are meaningful-// delimiter values.-func containsEmpty(values []any) bool {- for _, value := range values {- if s, ok := value.(string); ok && s == "" {- return true- }- }- return false-}-func validateDataOperationsParams(component string, params map[string]any, draft bool) error {for _, field := range []string{"select_keys", "remove_keys", "updates", "rename_keys", "filter_values"} {raw := params[field]diff --git a/internal/agent/component/dynamic_params_test.go b/internal/agent/component/dynamic_params_test.goindex 4b32381508..69c15bcc8b 100644--- a/internal/agent/component/dynamic_params_test.go+++ b/internal/agent/component/dynamic_params_test.go@@ -101,7 +101,7 @@ func TestValidateDynamicEntries(t *testing.T) {case "invoke variable row not a map", "invoke variable nil row", "invoke variable with numeric key", "invoke variables not an array", "invoke variables object":invokeDraft = false}- if test.component["component_name"] == "Message" || test.component["component_name"] == "DataOperations" || test.component["component_name"] == "Compiler" || test.component["component_name"] == "Loop" || test.component["component_name"] == "Iteration" || invokeDraft || test.name == "input option" {+ if test.component["component_name"] == "Message" || test.component["component_name"] == "DataOperations" || test.component["component_name"] == "Compiler" || test.component["component_name"] == "Loop" || test.component["component_name"] == "Iteration" || test.component["component_name"] == "TokenChunker" || invokeDraft || test.name == "input option" {if draftErr != nil {t.Fatalf("draft rejected unfinished entries: %v", draftErr)}diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex fa4688e649..517ceeeede 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -143,6 +143,9 @@ type TokenChunkerComponent struct {// Errors here surface as canvas compile failures (mirrors the// python check() phase).func NewTokenChunker(params map[string]any) (runtime.Component, error) {+ if err := schema.ValidateDelimiterEntries(params, false); err != nil {+ return nil, runtime.NewUserFacingError(fmt.Sprintf("TokenChunker: %s", err))+ }p := defaultsToken(tokenChunkerParam{})p.Update(params)if err := p.TokenChunkerParam.Validate(); err != nil {diff --git a/internal/ingestion/component/schema/delimiters.go b/internal/ingestion/component/schema/delimiters.gonew file mode 100644index 0000000000..c68659f73d--- /dev/null+++ b/internal/ingestion/component/schema/delimiters.go@@ -0,0 +1,37 @@+package schema++import "fmt"++// ValidateDelimiterEntries permits unfinished draft rows but requires string arrays.+// Whitespace is a meaningful split token, so only zero-length strings are incomplete.+func ValidateDelimiterEntries(params map[string]any, draft bool) error {+ for _, field := range []string{"delimiters", "children_delimiters"} {+ raw, present := params[field]+ if !present || raw == nil {+ continue+ }+ var values []string+ switch entries := raw.(type) {+ case []string:+ values = entries+ case []any:+ for _, entry := range entries {+ value, ok := entry.(string)+ if !ok {+ return fmt.Errorf("%s must contain strings", field)+ }+ values = append(values, value)+ }+ default:+ return fmt.Errorf("%s must be an array", field)+ }+ if !draft {+ for _, value := range values {+ if value == "" {+ return fmt.Errorf("%s does not support empty entries", field)+ }+ }+ }+ }+ return nil+}diff --git a/web/src/pages/agent/utils.test.ts b/web/src/pages/agent/utils.test.tsindex 5247444360..b9f0e1c742 100644--- a/web/src/pages/agent/utils.test.ts+++ b/web/src/pages/agent/utils.test.ts@@ -272,7 +272,7 @@ describe('transformTokenChunkerParams', () => {} as any);expect(result.overlapped_percent).toBeCloseTo(0.3);- expect(result.delimiters).toEqual(['\n']);+ expect(result.delimiters).toEqual(['\n', '']);expect(result.children_delimiters).toEqual(['|']);expect(result.table_context_size).toBe(81);expect(result.image_context_size).toBe(81);diff --git a/web/src/pages/agent/utils.ts b/web/src/pages/agent/utils.tsindex c47a91feec..43631a4566 100644--- a/web/src/pages/agent/utils.ts+++ b/web/src/pages/agent/utils.ts@@ -390,13 +390,17 @@ export function transformTokenChunkerParams(// while merging everything into a single chunk, but zeroing them here// wiped the user's settings on every save (they reloaded as 0 / ["\n"]).overlapped_percent: Number(rest.overlapped_percent) / 100,- delimiters: transformObjectArrayToPureArray(rest.delimiters, 'value'),+ delimiters: rest.delimiters.map(+ ({ value }: { value?: string }) => value ?? '',+ ),table_context_size: imageTableContextWindow,image_context_size: imageTableContextWindow,// Unset children delimiters if this option is not enabledchildren_delimiters: rest.enable_children- ? transformObjectArrayToPureArray(rest.children_delimiters, 'value')+ ? rest.children_delimiters.map(+ ({ value }: { value?: string }) => value ?? '',+ ): [],};}diff --git a/web/src/pages/agent/utils/canvas-checklist.ts b/web/src/pages/agent/utils/canvas-checklist.tsindex 2e95afaad8..5b2ed0fd31 100644--- a/web/src/pages/agent/utils/canvas-checklist.ts+++ b/web/src/pages/agent/utils/canvas-checklist.ts@@ -670,6 +670,27 @@ export function collectCanvasIssues({});}+ if (+ ['delimiters', 'children_delimiters'].some(+ (field) =>+ Array.isArray(form?.[field]) &&+ form[field].some((entry: unknown) => {+ const value =+ entry && typeof entry === 'object' && 'value' in entry+ ? entry.value+ : entry;+ return typeof value !== 'string' || value === '';+ }),+ )+ ) {+ issues.push({+ ...target,+ type: CanvasIssueType.MissingRequired,+ messageKey: 'flow.nodeFormInvalid',+ blocksRun: true,+ });+ }+if (ApiKeyRequiredOperators.includes(label) && !form?.api_key) {issues.push({...target,