Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(dataset): align parser methods order with python (#18877)

Sbaaoui Idriss9fe0aac8325d
diff --git a/internal/ingestion/pipeline/builtin_registry.go b/internal/ingestion/pipeline/builtin_registry.go
index d2381b771c..13ef7793e5 100644
--- a/internal/ingestion/pipeline/builtin_registry.go
+++ b/internal/ingestion/pipeline/builtin_registry.go
@@ -15,6 +15,24 @@ import (
const builtinTemplatePrefix = "ingestion_pipeline_"
+// builtinParserOrder matches Python's settings.PARSERS order, projected onto
+// the canonical builtin templates shipped by Go. Templates not in this list
+// are appended deterministically after the shared parser methods.
+var builtinParserOrder = map[string]int{
+ "general": 0,
+ "qa": 1,
+ "manual": 2,
+ "table": 3,
+ "paper": 4,
+ "book": 5,
+ "laws": 6,
+ "presentation": 7,
+ "picture": 8,
+ "one": 9,
+ "audio": 10,
+ "email": 11,
+}
+
//go:embed template/ingestion_pipeline_*.json
var builtinTemplateFS embed.FS
@@ -100,7 +118,17 @@ func NewRegistryFromFS(files fs.FS, dir string) (*Registry, error) {
meta := tpl.BuiltinPipelineMeta
list = append(list, &meta)
}
- sort.Slice(list, func(i, j int) bool { return list[i].ParserID < list[j].ParserID })
+ sort.SliceStable(list, func(i, j int) bool {
+ iOrder, iKnown := builtinParserOrder[list[i].ParserID]
+ jOrder, jKnown := builtinParserOrder[list[j].ParserID]
+ if iKnown && jKnown {
+ return iOrder < jOrder
+ }
+ if iKnown != jKnown {
+ return iKnown
+ }
+ return list[i].ParserID < list[j].ParserID
+ })
return &Registry{templates: templates, aliases: builtinAliases(), list: list}, nil
}
diff --git a/internal/ingestion/pipeline/builtin_registry_test.go b/internal/ingestion/pipeline/builtin_registry_test.go
index 09e38d3fea..6e499ea63c 100644
--- a/internal/ingestion/pipeline/builtin_registry_test.go
+++ b/internal/ingestion/pipeline/builtin_registry_test.go
@@ -58,8 +58,35 @@ func TestRegistryRefsMatchList(t *testing.T) {
if len(refs) != 2 {
t.Fatalf("len(Refs()) = %d, want 2", len(refs))
}
- if refs[0] != "book" || refs[1] != "general" {
- t.Fatalf("refs = %#v, want sorted builtin refs", refs)
+ if refs[0] != "general" || refs[1] != "book" {
+ t.Fatalf("refs = %#v, want Python-compatible builtin order", refs)
+ }
+}
+
+func TestRegistryCanonicalOrderAppendsUnknownTemplates(t *testing.T) {
+ dir := t.TempDir()
+ mustWrite := func(name, body string) {
+ if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600); err != nil {
+ t.Fatalf("write %s: %v", name, err)
+ }
+ }
+ for _, id := range []string{"email", "book", "general", "qa", "manual", "table", "paper", "laws", "presentation", "picture", "one", "audio", "custom"} {
+ mustWrite("ingestion_pipeline_"+id+".json", `{"title":{"en":"`+id+`"},"dsl":{"components":{}}}`)
+ }
+
+ r, err := NewRegistryFromDir(dir)
+ if err != nil {
+ t.Fatalf("NewRegistryFromDir: %v", err)
+ }
+ want := []string{"general", "qa", "manual", "table", "paper", "book", "laws", "presentation", "picture", "one", "audio", "email", "custom"}
+ if got := r.Refs(); len(got) != len(want) {
+ t.Fatalf("Refs() length = %d, want %d (%#v)", len(got), len(want), want)
+ } else {
+ for i := range want {
+ if got[i] != want[i] {
+ t.Fatalf("Refs()[%d] = %q, want %q; full order %#v", i, got[i], want[i], got)
+ }
+ }
}
}
diff --git a/test/testcases/restful_api/test_datasets.py b/test/testcases/restful_api/test_datasets.py
index bd3ac347b7..366eb87ba9 100644
--- a/test/testcases/restful_api/test_datasets.py
+++ b/test/testcases/restful_api/test_datasets.py
@@ -915,7 +915,7 @@ def test_dataset_update_chunk_method_invalid_contract(rest_client, clear_dataset
elif IS_GO_PROXY and chunk_method == "":
assert payload["message"] == "parser_id is required when parse_type is BuiltIn", payload
elif IS_GO_PROXY:
- assert payload["message"].startswith("input should be 'audio', 'book'") and payload["message"].endswith("or 'table'"), payload
+ assert payload["message"].startswith("input should be 'general', 'qa'") and payload["message"].endswith("or 'email'"), payload
else:
assert expected_chunk_message in payload["message"], payload
@@ -925,7 +925,7 @@ def test_dataset_update_chunk_method_invalid_contract(rest_client, clear_dataset
_skip_go_ignored_null(none_payload, PARSER_ID_FIELD)
assert none_payload["code"] == ARGUMENT_ERROR_CODE, none_payload
if IS_GO_PROXY:
- assert none_payload["message"].startswith("input should be 'audio', 'book'") and none_payload["message"].endswith("or 'table'"), none_payload
+ assert none_payload["message"].startswith("input should be 'general', 'qa'") and none_payload["message"].endswith("or 'email'"), none_payload
else:
assert expected_chunk_message in none_payload["message"], none_payload
@@ -1686,7 +1686,7 @@ def test_dataset_create_permission_and_chunk_method_contract(rest_client, clear_
elif IS_GO_PROXY and chunk_method == "":
assert payload["message"] == "parser_id is required when parse_type is BuiltIn", payload
elif IS_GO_PROXY:
- assert payload["message"].startswith("input should be 'audio', 'book'") and payload["message"].endswith("or 'table'"), payload
+ assert payload["message"].startswith("input should be 'general', 'qa'") and payload["message"].endswith("or 'email'"), payload
else:
assert expected_chunk_message in payload["message"], payload
diff --git a/web/src/hooks/use-user-setting-request.tsx b/web/src/hooks/use-user-setting-request.tsx
index a5de23669c..3e9d4ca734 100644
--- a/web/src/hooks/use-user-setting-request.tsx
+++ b/web/src/hooks/use-user-setting-request.tsx
@@ -175,36 +175,7 @@ export const useSelectParserList = (): Array<{
const translated = t(key);
return translated !== key ? translated : title;
};
- // Order the Go pipeline catalog the same way as the Python
- // backend's parser_ids (settings.PARSERS), so the chunk-method
- // dropdown lists commonly used methods first. The Python "naive"
- // slot corresponds to the Go "general" pipeline. Go-only pipelines
- // Any backend-only pipelines keep their API order at the end.
- const pythonParserOrder = [
- 'general', // "naive" in Python parser_ids
- 'qa',
- 'resume',
- 'manual',
- 'table',
- 'paper',
- 'book',
- 'laws',
- 'presentation',
- 'picture',
- 'one',
- 'audio',
- 'email',
- 'tag',
- ];
- const order = new Map(
- pythonParserOrder.map((id, index) => [id, index]),
- );
- const sortedList = [...pipelineList].sort(
- (a, b) =>
- (order.get(a.id) ?? pythonParserOrder.length) -
- (order.get(b.id) ?? pythonParserOrder.length),
- );
- return sortedList.map((item) => ({
+ return pipelineList.map((item) => ({
value: item.id,
label: labelFromAPI(item.id, item.title),
}));