RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(compiler): keep LLM runtime params when saving a document pipeline config (#19554)
euvrea230f1e69432
diff --git a/internal/ingestion/pipeline/pipeline_params.go b/internal/ingestion/pipeline/pipeline_params.goindex 08d75876c2..9a4ec49172 100644--- a/internal/ingestion/pipeline/pipeline_params.go+++ b/internal/ingestion/pipeline/pipeline_params.go@@ -22,6 +22,8 @@ var llmRuntimeParamKeys = map[string]struct{}{"presence_penalty": {},"frequency_penalty": {},"outputs": {},+ "parameter": {},+ "thinking": {},"temperatureEnabled": {},"topPEnabled": {},"presencePenaltyEnabled": {},@@ -74,11 +76,23 @@ func isExtractorComponent(cpnID, componentName string) bool {strings.HasPrefix(lowerID, "extractor_")}+// isCompilerComponent returns true if the component ID indicates a Compiler+// component, whose LLM-backed runtime parameters follow the same surface as+// the Extractor's.+func isCompilerComponent(cpnID string) bool {+ lowerID := strings.ToLower(cpnID)+ return strings.HasPrefix(lowerID, "compiler:") ||+ strings.HasPrefix(lowerID, "compiler_")+}+// getComponentParamWhitelist returns the dynamic parameter whitelist for a component type.func getComponentParamWhitelist(cpnID string) (map[string]struct{}, bool) {if isExtractorComponent(cpnID, "") {return extractorValidParamKeys, true}+ if isCompilerComponent(cpnID) {+ return llmRuntimeParamKeys, true+ }return nil, false}@@ -130,7 +144,7 @@ func CleanComponentParams(dslJSON []byte, rawConfig map[string]interface{}) map[params = normalizeGeneralComponentParams(params)}dynamicWhitelist, hasDynamic := getComponentParamWhitelist(key)- if hasDynamic {+ if isExtractorComponent(key, "") {params = NormalizeExtractorParams(params)}cleaned := make(map[string]any, len(params))diff --git a/internal/ingestion/pipeline/pipeline_params_test.go b/internal/ingestion/pipeline/pipeline_params_test.goindex 33422be57d..08dfff7867 100644--- a/internal/ingestion/pipeline/pipeline_params_test.go+++ b/internal/ingestion/pipeline/pipeline_params_test.go@@ -661,3 +661,107 @@ func TestNormalizeExtractorParams_TableDriven(t *testing.T) {})}}++// compilerDSL mirrors the compiler pipeline template's component structure.+func compilerDSL(t *testing.T) []byte {+ t.Helper()+ dsl := map[string]any{+ "components": map[string]any{+ "Compiler:NewBoxesLove": map[string]any{+ "obj": map[string]any{+ "component_name": "Compiler",+ "params": map[string]any{+ "outputs": map[string]any{},+ "compilation_template_group_id": "",+ "llm_id": "",+ "plan": false,+ },+ },+ },+ },+ }+ raw, err := json.Marshal(dsl)+ if err != nil {+ t.Fatalf("marshal dsl fixture: %v", err)+ }+ return raw+}++func TestCleanComponentParams_CompilerKeepsLLMRuntimeParams(t *testing.T) {+ dslJSON := compilerDSL(t)+ raw := map[string]any{+ "Compiler:NewBoxesLove": map[string]any{+ "compilation_template_group_id": "group-1",+ "llm_id": "model-1",+ "temperature": 0.7,+ "temperatureEnabled": true,+ "top_p": 0.9,+ "topPEnabled": true,+ "presence_penalty": 0.2,+ "presencePenaltyEnabled": true,+ "frequency_penalty": 0.3,+ "frequencyPenaltyEnabled": true,+ "max_tokens": 1024.0,+ "maxTokensEnabled": true,+ "parameter": "Custom",+ "thinking": "enabled",+ "mode": "structure",+ },+ }+ result := CleanComponentParams(dslJSON, raw)+ params, ok := result["Compiler:NewBoxesLove"].(map[string]any)+ if !ok {+ t.Fatalf("expected Compiler params to survive, got %#v", result)+ }+ for _, key := range []string{+ "compilation_template_group_id", "llm_id",+ "temperature", "temperatureEnabled",+ "top_p", "topPEnabled",+ "presence_penalty", "presencePenaltyEnabled",+ "frequency_penalty", "frequencyPenaltyEnabled",+ "max_tokens", "maxTokensEnabled",+ "parameter", "thinking",+ } {+ if _, ok := params[key]; !ok {+ t.Errorf("expected LLM runtime param %q to be kept, got %#v", key, params)+ }+ }+ if _, ok := params["mode"]; ok {+ t.Error("expected unknown param key 'mode' to be dropped")+ }+ for _, key := range []string{"keywords", "questions", "tags", "summary", "metadata"} {+ if _, ok := params[key]; ok {+ t.Errorf("expected no extractor group %q on Compiler params", key)+ }+ }+}++func TestBuildParserConfig_CompilerRuntimeParamsSurvive(t *testing.T) {+ dslJSON := compilerDSL(t)+ raw := map[string]any{+ "Compiler:NewBoxesLove": map[string]any{+ "compilation_template_group_id": "group-1",+ "llm_id": "model-1",+ "temperature": 0.7,+ "temperatureEnabled": true,+ "thinking": "disabled",+ },+ }+ result := BuildParserConfig(dslJSON, raw)+ params, ok := result["Compiler:NewBoxesLove"].(map[string]any)+ if !ok {+ t.Fatalf("expected Compiler entry in built parser_config, got %#v", result)+ }+ if params["compilation_template_group_id"] != "group-1" || params["llm_id"] != "model-1" {+ t.Errorf("compiler overrides mismatch: %#v", params)+ }+ if params["temperature"] != 0.7 || params["temperatureEnabled"] != true {+ t.Errorf("expected LLM runtime overrides to survive, got %#v", params)+ }+ if params["thinking"] != "disabled" {+ t.Errorf("expected thinking override to survive, got %#v", params)+ }+ if plan, ok := params["plan"].(bool); !ok || plan {+ t.Errorf("expected DSL default plan=false to be baked in, got %#v", params["plan"])+ }+}diff --git a/web/src/pages/agent/constant/pipeline.tsx b/web/src/pages/agent/constant/pipeline.tsxindex 131a7cdc38..605b5b8440 100644--- a/web/src/pages/agent/constant/pipeline.tsx+++ b/web/src/pages/agent/constant/pipeline.tsx@@ -424,6 +424,7 @@ export function getInitialExtractorValues() {}export const initialCompilationValues = {+ ...initialLlmBaseValues,compilation_template_group_id: '',llm_id: '',outputs: {diff --git a/web/src/pages/agent/form/compilation-form/index.tsx b/web/src/pages/agent/form/compilation-form/index.tsxindex 9a757654a2..90829e7dab 100644--- a/web/src/pages/agent/form/compilation-form/index.tsx+++ b/web/src/pages/agent/form/compilation-form/index.tsx@@ -1,5 +1,8 @@import { CompilationTemplateFormField } from '@/components/compilation-template-form-field';-import { LargeModelFormField } from '@/components/large-model-form-field';+import {+ LlmSettingFieldItems,+ LlmSettingSchema,+} from '@/components/llm-setting-items/next';import { useSyncExternalFormErrors } from '@/components/pipeline-operator-tabs/use-sync-external-form-errors';import { Form } from '@/components/ui/form';import { zodResolver } from '@hookform/resolvers/zod';@@ -23,7 +26,7 @@ export function buildCompilationFormSchema(t: TFunction) {compilation_template_group_id: z.string().min(1, t('knowledgeConfiguration.compilationTemplateRequired')),- llm_id: z.string().optional(),+ ...LlmSettingSchema,});}@@ -63,10 +66,9 @@ const CompilationForm = ({<Form {...form}><FormWrapper><CompilationTemplateFormField name="compilation_template_group_id"></CompilationTemplateFormField>- <LargeModelFormField- name="llm_id"+ <LlmSettingFieldItemsownerTenantId={ownerTenantId}- ></LargeModelFormField>+ ></LlmSettingFieldItems></FormWrapper>{!hideOutputs && (<div className="p-5">diff --git a/web/src/pages/agent/utils.ts b/web/src/pages/agent/utils.tsindex 4fce126df6..57e3f61b90 100644--- a/web/src/pages/agent/utils.ts+++ b/web/src/pages/agent/utils.ts@@ -436,10 +436,11 @@ export function transformTitleChunkerParams(};}-// LLM setting keys the Go extractor DSL keeps besides the nested groups.-// Mirrors LlmSettingSchema (components/llm-setting-items/next) — duplicated-// here as a plain list so this module doesn't import the form components.-const ExtractorLlmSettingKeys = [+// LLM setting keys LLM-backed pipeline operators (Extractor, Compiler) keep in+// their DSL params. Mirrors LlmSettingSchema (components/llm-setting-items/next)+// — duplicated here as a plain list so this module doesn't import the form+// components.+export const LlmSettingParamKeys = ['llm_id','temperature','top_p',@@ -514,7 +515,7 @@ function transformExtractorParamsGo(// the LLM settings the form defines — no legacy flat mirrors, no// display-only fields like outputs.return {- ...pick(params, ExtractorLlmSettingKeys),+ ...pick(params, LlmSettingParamKeys),keywords: {top_n: keywordsTopN,system_prompt: keywordsSysPrompt,@@ -548,6 +549,17 @@ export function transformExtractorParams(})(params);}+// The Compiler reads the compilation template group plus the same LLM runtime+// settings as the Extractor; display-only fields like outputs are dropped.+export function transformCompilationParams(+ params: Record<string, any>,+): Record<string, any> {+ return {+ compilation_template_group_id: params?.compilation_template_group_id,+ ...pick(params, LlmSettingParamKeys),+ };+}+function transformDataOperationsParams(params: DataOperationsFormSchemaType) {return {...params,diff --git a/web/src/pages/agent/utils/__tests__/compilation-transform.test.ts b/web/src/pages/agent/utils/__tests__/compilation-transform.test.tsnew file mode 100644index 0000000000..b33ca37ae1--- /dev/null+++ b/web/src/pages/agent/utils/__tests__/compilation-transform.test.ts@@ -0,0 +1,63 @@+import { Operator } from '@/constants/agent';+import { transformFormConfigToApi } from '@/utils/pipeline-operator';+import { transformCompilationParams } from '../../utils';++describe('Compiler parameter transformations', () => {+ it('emits the compilation template group plus the LLM runtime settings', () => {+ const input: any = {+ compilation_template_group_id: 'group-1',+ llm_id: 'model-1',+ temperature: 0.5,+ temperatureEnabled: true,+ top_p: 0.9,+ topPEnabled: true,+ presence_penalty: 0.2,+ presencePenaltyEnabled: true,+ frequency_penalty: 0.3,+ frequencyPenaltyEnabled: true,+ max_tokens: 1024,+ maxTokensEnabled: true,+ parameter: 'Custom',+ thinking: 'enabled',+ // Fields that must not leak into the DSL params+ outputs: { chunks: { type: 'Array<Object>', value: [] } },+ mode: 'structure',+ };++ const result = transformCompilationParams(input);++ expect(result).toEqual({+ compilation_template_group_id: 'group-1',+ llm_id: 'model-1',+ temperature: 0.5,+ temperatureEnabled: true,+ top_p: 0.9,+ topPEnabled: true,+ presence_penalty: 0.2,+ presencePenaltyEnabled: true,+ frequency_penalty: 0.3,+ frequencyPenaltyEnabled: true,+ max_tokens: 1024,+ maxTokensEnabled: true,+ parameter: 'Custom',+ thinking: 'enabled',+ });+ });++ it('is wired into the document pipeline save path', () => {+ const result = transformFormConfigToApi(Operator.Compiler, {+ compilation_template_group_id: 'group-1',+ llm_id: 'model-1',+ temperature: 0.7,+ temperatureEnabled: true,+ outputs: {},+ });++ expect(result).toEqual({+ compilation_template_group_id: 'group-1',+ llm_id: 'model-1',+ temperature: 0.7,+ temperatureEnabled: true,+ });+ });+});diff --git a/web/src/utils/pipeline-operator.ts b/web/src/utils/pipeline-operator.tsindex 25fb249fe2..2295be26b3 100644--- a/web/src/utils/pipeline-operator.ts+++ b/web/src/utils/pipeline-operator.ts@@ -18,6 +18,7 @@ import { Operator } from '@/constants/agent';import { DSL, RAGFlowNodeType } from '@/interfaces/database/agent';import {getInitialExtractorValues,+ initialCompilationValues,initialGoExtractorValues,initialGeneralChunkerValues,initialParserValues,@@ -26,6 +27,7 @@ import {initialTokenizerValues,} from '@/pages/agent/constant/pipeline';import {+ transformCompilationParams,transformExtractorParams,transformGeneralChunkerParams,transformParserParams,@@ -395,6 +397,8 @@ export function transformFormConfigToApi(return transformParserParams(config as any);case Operator.Extractor:return transformExtractorParams(config as any);+ case Operator.Compiler:+ return transformCompilationParams(config as Record<string, any>);case Operator.Tokenizer:return config; // passthrough for Tokenizercase Operator.TokenChunker:@@ -444,6 +448,11 @@ export function normalizeOperatorForm(...cloneDeep(getInitialExtractorValues()),...rawForm,};+ case Operator.Compiler:+ return {+ ...cloneDeep(initialCompilationValues),+ ...rawForm,+ };case Operator.Tokenizer:return {...cloneDeep(initialTokenizerValues),