Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(agent): save incomplete chunker delimiters and gate Run (#21371)

Sbaaoui Idrissba9bbffcd168
diff --git a/internal/agent/component/dynamic_params.go b/internal/agent/component/dynamic_params.go
index 7833486f2b..c77c28e860 100644
--- a/internal/agent/component/dynamic_params.go
+++ b/internal/agent/component/dynamic_params.go
@@ -7,6 +7,7 @@ import (
"strings"
"ragflow/internal/agent/tool"
+ "ragflow/internal/ingestion/component/schema"
)
// ErrCompilationTemplateRequired identifies a Compiler component without a
@@ -20,7 +21,7 @@ func ValidateDynamicEntries(dsl map[string]any) error {
// ValidateDraftDynamicEntries allows unfinished Message content, DataOperations
// entries, Invoke mappings, Compiler configuration, input options, Loop settings,
-// Iteration outputs, SQL connections, and search filters in saved drafts.
+// Iteration outputs, SQL connections, search filters, and chunker delimiters in saved drafts.
// All other checks remain unchanged.
func ValidateDraftDynamicEntries(dsl map[string]any) error {
return validateDynamicEntries(dsl, true)
@@ -55,13 +56,8 @@ func validateDynamicParams(component string, params map[string]any, draft bool)
if err := tool.ValidateSearchFilters(params, draft); err != nil {
return fmt.Errorf("[%s] %w", component, err)
}
- // Delimiter entries are split tokens, not names: whitespace-only values
- // ("\n", "\t") are valid delimiters — the chunker form's default row is a
- // real newline — so only a zero-length string marks an incomplete row.
- for _, field := range []string{"delimiters", "children_delimiters"} {
- if values, ok := params[field].([]any); ok && containsEmpty(values) {
- return fmt.Errorf("[%s] %s does not support empty entries", component, field)
- }
+ if err := schema.ValidateDelimiterEntries(params, draft); err != nil {
+ return fmt.Errorf("[%s] %w", component, err)
}
switch strings.ToLower(component) {
case "message":
@@ -292,18 +288,6 @@ func containsBlank(values []any) bool {
return false
}
-// containsEmpty reports whether any entry is a zero-length string. Unlike
-// containsBlank it keeps whitespace-only entries, which are meaningful
-// delimiter values.
-func containsEmpty(values []any) bool {
- for _, value := range values {
- if s, ok := value.(string); ok && s == "" {
- return true
- }
- }
- return false
-}
-
func validateDataOperationsParams(component string, params map[string]any, draft bool) error {
for _, field := range []string{"select_keys", "remove_keys", "updates", "rename_keys", "filter_values"} {
raw := params[field]
diff --git a/internal/agent/component/dynamic_params_test.go b/internal/agent/component/dynamic_params_test.go
index 4b32381508..69c15bcc8b 100644
--- a/internal/agent/component/dynamic_params_test.go
+++ b/internal/agent/component/dynamic_params_test.go
@@ -101,7 +101,7 @@ func TestValidateDynamicEntries(t *testing.T) {
case "invoke variable row not a map", "invoke variable nil row", "invoke variable with numeric key", "invoke variables not an array", "invoke variables object":
invokeDraft = false
}
- if test.component["component_name"] == "Message" || test.component["component_name"] == "DataOperations" || test.component["component_name"] == "Compiler" || test.component["component_name"] == "Loop" || test.component["component_name"] == "Iteration" || invokeDraft || test.name == "input option" {
+ if test.component["component_name"] == "Message" || test.component["component_name"] == "DataOperations" || test.component["component_name"] == "Compiler" || test.component["component_name"] == "Loop" || test.component["component_name"] == "Iteration" || test.component["component_name"] == "TokenChunker" || invokeDraft || test.name == "input option" {
if draftErr != nil {
t.Fatalf("draft rejected unfinished entries: %v", draftErr)
}
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index fa4688e649..517ceeeede 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -143,6 +143,9 @@ type TokenChunkerComponent struct {
// Errors here surface as canvas compile failures (mirrors the
// python check() phase).
func NewTokenChunker(params map[string]any) (runtime.Component, error) {
+ if err := schema.ValidateDelimiterEntries(params, false); err != nil {
+ return nil, runtime.NewUserFacingError(fmt.Sprintf("TokenChunker: %s", err))
+ }
p := defaultsToken(tokenChunkerParam{})
p.Update(params)
if err := p.TokenChunkerParam.Validate(); err != nil {
diff --git a/internal/ingestion/component/schema/delimiters.go b/internal/ingestion/component/schema/delimiters.go
new file mode 100644
index 0000000000..c68659f73d
--- /dev/null
+++ b/internal/ingestion/component/schema/delimiters.go
@@ -0,0 +1,37 @@
+package schema
+
+import "fmt"
+
+// ValidateDelimiterEntries permits unfinished draft rows but requires string arrays.
+// Whitespace is a meaningful split token, so only zero-length strings are incomplete.
+func ValidateDelimiterEntries(params map[string]any, draft bool) error {
+ for _, field := range []string{"delimiters", "children_delimiters"} {
+ raw, present := params[field]
+ if !present || raw == nil {
+ continue
+ }
+ var values []string
+ switch entries := raw.(type) {
+ case []string:
+ values = entries
+ case []any:
+ for _, entry := range entries {
+ value, ok := entry.(string)
+ if !ok {
+ return fmt.Errorf("%s must contain strings", field)
+ }
+ values = append(values, value)
+ }
+ default:
+ return fmt.Errorf("%s must be an array", field)
+ }
+ if !draft {
+ for _, value := range values {
+ if value == "" {
+ return fmt.Errorf("%s does not support empty entries", field)
+ }
+ }
+ }
+ }
+ return nil
+}
diff --git a/web/src/pages/agent/utils.test.ts b/web/src/pages/agent/utils.test.ts
index 5247444360..b9f0e1c742 100644
--- a/web/src/pages/agent/utils.test.ts
+++ b/web/src/pages/agent/utils.test.ts
@@ -272,7 +272,7 @@ describe('transformTokenChunkerParams', () => {
} as any);
expect(result.overlapped_percent).toBeCloseTo(0.3);
- expect(result.delimiters).toEqual(['\n']);
+ expect(result.delimiters).toEqual(['\n', '']);
expect(result.children_delimiters).toEqual(['|']);
expect(result.table_context_size).toBe(81);
expect(result.image_context_size).toBe(81);
diff --git a/web/src/pages/agent/utils.ts b/web/src/pages/agent/utils.ts
index c47a91feec..43631a4566 100644
--- a/web/src/pages/agent/utils.ts
+++ b/web/src/pages/agent/utils.ts
@@ -390,13 +390,17 @@ export function transformTokenChunkerParams(
// while merging everything into a single chunk, but zeroing them here
// wiped the user's settings on every save (they reloaded as 0 / ["\n"]).
overlapped_percent: Number(rest.overlapped_percent) / 100,
- delimiters: transformObjectArrayToPureArray(rest.delimiters, 'value'),
+ delimiters: rest.delimiters.map(
+ ({ value }: { value?: string }) => value ?? '',
+ ),
table_context_size: imageTableContextWindow,
image_context_size: imageTableContextWindow,
// Unset children delimiters if this option is not enabled
children_delimiters: rest.enable_children
- ? transformObjectArrayToPureArray(rest.children_delimiters, 'value')
+ ? rest.children_delimiters.map(
+ ({ value }: { value?: string }) => value ?? '',
+ )
: [],
};
}
diff --git a/web/src/pages/agent/utils/canvas-checklist.ts b/web/src/pages/agent/utils/canvas-checklist.ts
index 2e95afaad8..5b2ed0fd31 100644
--- a/web/src/pages/agent/utils/canvas-checklist.ts
+++ b/web/src/pages/agent/utils/canvas-checklist.ts
@@ -670,6 +670,27 @@ export function collectCanvasIssues({
});
}
+ if (
+ ['delimiters', 'children_delimiters'].some(
+ (field) =>
+ Array.isArray(form?.[field]) &&
+ form[field].some((entry: unknown) => {
+ const value =
+ entry && typeof entry === 'object' && 'value' in entry
+ ? entry.value
+ : entry;
+ return typeof value !== 'string' || value === '';
+ }),
+ )
+ ) {
+ issues.push({
+ ...target,
+ type: CanvasIssueType.MissingRequired,
+ messageKey: 'flow.nodeFormInvalid',
+ blocksRun: true,
+ });
+ }
+
if (ApiKeyRequiredOperators.includes(label) && !form?.api_key) {
issues.push({
...target,