RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(dataset): align parser methods order with python (#18877)
Sbaaoui Idriss9fe0aac8325d
diff --git a/internal/ingestion/pipeline/builtin_registry.go b/internal/ingestion/pipeline/builtin_registry.goindex d2381b771c..13ef7793e5 100644--- a/internal/ingestion/pipeline/builtin_registry.go+++ b/internal/ingestion/pipeline/builtin_registry.go@@ -15,6 +15,24 @@ import (const builtinTemplatePrefix = "ingestion_pipeline_"+// builtinParserOrder matches Python's settings.PARSERS order, projected onto+// the canonical builtin templates shipped by Go. Templates not in this list+// are appended deterministically after the shared parser methods.+var builtinParserOrder = map[string]int{+ "general": 0,+ "qa": 1,+ "manual": 2,+ "table": 3,+ "paper": 4,+ "book": 5,+ "laws": 6,+ "presentation": 7,+ "picture": 8,+ "one": 9,+ "audio": 10,+ "email": 11,+}+//go:embed template/ingestion_pipeline_*.jsonvar builtinTemplateFS embed.FS@@ -100,7 +118,17 @@ func NewRegistryFromFS(files fs.FS, dir string) (*Registry, error) {meta := tpl.BuiltinPipelineMetalist = append(list, &meta)}- sort.Slice(list, func(i, j int) bool { return list[i].ParserID < list[j].ParserID })+ sort.SliceStable(list, func(i, j int) bool {+ iOrder, iKnown := builtinParserOrder[list[i].ParserID]+ jOrder, jKnown := builtinParserOrder[list[j].ParserID]+ if iKnown && jKnown {+ return iOrder < jOrder+ }+ if iKnown != jKnown {+ return iKnown+ }+ return list[i].ParserID < list[j].ParserID+ })return &Registry{templates: templates, aliases: builtinAliases(), list: list}, nil}diff --git a/internal/ingestion/pipeline/builtin_registry_test.go b/internal/ingestion/pipeline/builtin_registry_test.goindex 09e38d3fea..6e499ea63c 100644--- a/internal/ingestion/pipeline/builtin_registry_test.go+++ b/internal/ingestion/pipeline/builtin_registry_test.go@@ -58,8 +58,35 @@ func TestRegistryRefsMatchList(t *testing.T) {if len(refs) != 2 {t.Fatalf("len(Refs()) = %d, want 2", len(refs))}- if refs[0] != "book" || refs[1] != "general" {- t.Fatalf("refs = %#v, want sorted builtin refs", refs)+ if refs[0] != "general" || refs[1] != "book" {+ t.Fatalf("refs = %#v, want Python-compatible builtin order", refs)+ }+}++func TestRegistryCanonicalOrderAppendsUnknownTemplates(t *testing.T) {+ dir := t.TempDir()+ mustWrite := func(name, body string) {+ if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600); err != nil {+ t.Fatalf("write %s: %v", name, err)+ }+ }+ for _, id := range []string{"email", "book", "general", "qa", "manual", "table", "paper", "laws", "presentation", "picture", "one", "audio", "custom"} {+ mustWrite("ingestion_pipeline_"+id+".json", `{"title":{"en":"`+id+`"},"dsl":{"components":{}}}`)+ }++ r, err := NewRegistryFromDir(dir)+ if err != nil {+ t.Fatalf("NewRegistryFromDir: %v", err)+ }+ want := []string{"general", "qa", "manual", "table", "paper", "book", "laws", "presentation", "picture", "one", "audio", "email", "custom"}+ if got := r.Refs(); len(got) != len(want) {+ t.Fatalf("Refs() length = %d, want %d (%#v)", len(got), len(want), want)+ } else {+ for i := range want {+ if got[i] != want[i] {+ t.Fatalf("Refs()[%d] = %q, want %q; full order %#v", i, got[i], want[i], got)+ }+ }}}diff --git a/test/testcases/restful_api/test_datasets.py b/test/testcases/restful_api/test_datasets.pyindex bd3ac347b7..366eb87ba9 100644--- a/test/testcases/restful_api/test_datasets.py+++ b/test/testcases/restful_api/test_datasets.py@@ -915,7 +915,7 @@ def test_dataset_update_chunk_method_invalid_contract(rest_client, clear_datasetelif IS_GO_PROXY and chunk_method == "":assert payload["message"] == "parser_id is required when parse_type is BuiltIn", payloadelif IS_GO_PROXY:- assert payload["message"].startswith("input should be 'audio', 'book'") and payload["message"].endswith("or 'table'"), payload+ assert payload["message"].startswith("input should be 'general', 'qa'") and payload["message"].endswith("or 'email'"), payloadelse:assert expected_chunk_message in payload["message"], payload@@ -925,7 +925,7 @@ def test_dataset_update_chunk_method_invalid_contract(rest_client, clear_dataset_skip_go_ignored_null(none_payload, PARSER_ID_FIELD)assert none_payload["code"] == ARGUMENT_ERROR_CODE, none_payloadif IS_GO_PROXY:- assert none_payload["message"].startswith("input should be 'audio', 'book'") and none_payload["message"].endswith("or 'table'"), none_payload+ assert none_payload["message"].startswith("input should be 'general', 'qa'") and none_payload["message"].endswith("or 'email'"), none_payloadelse:assert expected_chunk_message in none_payload["message"], none_payload@@ -1686,7 +1686,7 @@ def test_dataset_create_permission_and_chunk_method_contract(rest_client, clear_elif IS_GO_PROXY and chunk_method == "":assert payload["message"] == "parser_id is required when parse_type is BuiltIn", payloadelif IS_GO_PROXY:- assert payload["message"].startswith("input should be 'audio', 'book'") and payload["message"].endswith("or 'table'"), payload+ assert payload["message"].startswith("input should be 'general', 'qa'") and payload["message"].endswith("or 'email'"), payloadelse:assert expected_chunk_message in payload["message"], payloaddiff --git a/web/src/hooks/use-user-setting-request.tsx b/web/src/hooks/use-user-setting-request.tsxindex a5de23669c..3e9d4ca734 100644--- a/web/src/hooks/use-user-setting-request.tsx+++ b/web/src/hooks/use-user-setting-request.tsx@@ -175,36 +175,7 @@ export const useSelectParserList = (): Array<{const translated = t(key);return translated !== key ? translated : title;};- // Order the Go pipeline catalog the same way as the Python- // backend's parser_ids (settings.PARSERS), so the chunk-method- // dropdown lists commonly used methods first. The Python "naive"- // slot corresponds to the Go "general" pipeline. Go-only pipelines- // Any backend-only pipelines keep their API order at the end.- const pythonParserOrder = [- 'general', // "naive" in Python parser_ids- 'qa',- 'resume',- 'manual',- 'table',- 'paper',- 'book',- 'laws',- 'presentation',- 'picture',- 'one',- 'audio',- 'email',- 'tag',- ];- const order = new Map(- pythonParserOrder.map((id, index) => [id, index]),- );- const sortedList = [...pipelineList].sort(- (a, b) =>- (order.get(a.id) ?? pythonParserOrder.length) -- (order.get(b.id) ?? pythonParserOrder.length),- );- return sortedList.map((item) => ({+ return pipelineList.map((item) => ({value: item.id,label: labelFromAPI(item.id, item.title),}));