RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(extractor): unify 5-in-1 modular schema across dataset and pipeline, drop legacy field_name and custom prompt (#18432)
jay777212a1a93ed3abf
diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.goindex 417c61d9ff..63b11a7b43 100644--- a/internal/ingestion/component/extractor.go+++ b/internal/ingestion/component/extractor.go@@ -14,50 +14,20 @@// limitations under the License.//-// Package component — Extractor component (Phase 2.5 of-// port-rag-flow-pipeline-to-go.md §4 row 2.5).+// Package component — Extractor component.//-// SCOPE (honest):+// SCOPE://-// - PROVIDER-AGNOSTIC (§8 Q1): the Extractor does NOT depend on any+// - PROVIDER-AGNOSTIC: the Extractor does NOT depend on any// specific LLM provider. It dispatches every chat call through-// internal/entity/models — the same factory routes 48 of the 56-// Python ChatModel providers registered there (factory.go switch,-// lines 36-156). The 8 providers NOT yet in the Go switch (LeptonAI,-// Gemini LiteLLM path, PerfXCloud, 01.AI / Lingyi, DeerAPI,-// Astraflow-CN, RAGcon, New API) ARE unreachable from this-// component — an llm_id resolving to one of those falls through to-// NewDummyModel and the chat call returns a deterministic "dummy"-// response. We DO NOT panic: errors are surfaced as a clean-// "no driver for %q" wrap that callers can log and route.+// internal/entity/models.//-// - LLM CALL SHAPE: one chat call per chunk (no batching). LLM-// calls are inherently serial; sequential per-chunk processing-// keeps test ordering deterministic under -race.+// - CONCURRENCY: auto extraction tasks (keywords, questions, summary, metadata)+// run concurrently across chunks via a bounded worker pool (extractorPool).//// - TIMEOUT / ELAPSED: the call is wrapped in-// runtime.WithTimeout(60s) and runtime.TrackElapsed so the-// upstream pipeline gets _created_time / _elapsed_time stamps-// matching the python ProcessBase contract (base.py:42, 58).-//-// - JSON PARSING: the prompt asks the LLM to return a JSON object;-// we best-effort parse the response into map[string]any. A-// non-JSON response is NOT a hard error — it's surfaced as the-// raw string under the same field name so downstream callers-// can decide what to do.-//-// - WHAT IS NOT YET PORTED: the python _build_TOC branch-// (rag/flow/extractor/extractor.py:40-72) requires the TOC-// generator (rag.prompts.generator.run_toc_from_text). That-// service has no Go counterpart yet; the current Extractor-// short-circuits with a clear error when field_name == "toc"-// so a future Phase 2.5+ task can fill the gap without a-// silent regression.-//-// - SINGLE-CHUNK FAST PATH: when no chunk list is wired in,-// the LLM is called once with the resolved args directly (no-// chunk substitution). Matches python _invoke path-// (line 108: msg, sys_prompt = self._sys_prompt_and_msg([], args)).+// runtime.WithTimeout(600s) and runtime.TrackElapsed so the+// upstream pipeline gets _created_time / _elapsed_time stamps.package componentimport (@@ -167,12 +137,7 @@ Extract the most important keywords/phrases of a given piece of text content.- Summarize the text content, and give the top %d important keywords/phrases.- The keywords MUST be in the same language as the given piece of text content.- The keywords are delimited by ENGLISH COMMA.-- Output keywords ONLY.-------## Text Content-%s`+- Output keywords ONLY.`autoQuestionPrompt = `## RoleYou are a text analyzer.@@ -186,12 +151,7 @@ Propose questions about a given piece of text content.- The questions SHOULD cover the main content of the text as much as possible.- The questions MUST be in the same language as the given piece of text content.- One question per line.-- Output questions ONLY.-------## Text Content-%s`+- Output questions ONLY.`autoMetadataPrompt = `## Role: Metadata extraction expert.## Rules:@@ -203,9 +163,6 @@ Propose questions about a given piece of text content.- Output: ONLY a valid JSON string. No Markdown, no notes.## Schema for extraction:-%s--## Content to analyze:%s`autoSummaryPrompt = `## Role@@ -239,10 +196,12 @@ type ExtractorComponent struct {// Param map shape (all keys optional; missing → Defaults())://// {-// "field_name": string, — optional; key the extraction lands under// "llm_id": string, — optional; resolves via models.NewModelFactory-// "system_prompt": string, — optional override-// "prompt": string, — optional user prompt+// "keywords": map[string]any, — optional auto keywords extraction config+// "questions": map[string]any, — optional auto questions extraction config+// "tags": map[string]any, — optional auto tags extraction config+// "summary": map[string]any, — optional auto summary extraction config+// "metadata": map[string]any, — optional auto metadata extraction config// }//// errors here surface as canvas compile failures so a malformed@@ -250,70 +209,31 @@ type ExtractorComponent struct {func NewExtractorComponent(params map[string]any) (runtime.Component, error) {p := schema.ExtractorParam{}.Defaults()if params != nil {- if v, ok := params["field_name"].(string); ok {- p.FieldName = v- }if v, ok := params["llm_id"].(string); ok {p.LLMID = v}- if v, ok := params["system_prompt"].(string); ok {- p.SystemPrompt = v- } else if v, ok := params["sys_prompt"].(string); ok {- p.SystemPrompt = v- }- if v, ok := params["prompt"].(string); ok {- p.Prompt = v- } else if v, ok := params["prompts"].(string); ok && v != "" {- // Python agent/component/llm.py:119-120 normalizes a bare-string- // prompts into [{"role":"user","content":prompts}]. Mirror that- // here so a front-end/template that emits prompts as a string- // (the graph.nodes form / dsl testdata) is not silently dropped- // by the .([]any) assertion on the list branch below.- p.Prompt = v- } else if promptsRaw, ok := params["prompts"].([]any); ok && len(promptsRaw) > 0 {- if first, ok := promptsRaw[0].(map[string]any); ok {- if content, ok := first["content"].(string); ok {- p.Prompt = content- }- }- }- // 1. Keywords (modular or flat)++ // 1. Keywordsif kwRaw, ok := params["keywords"].(map[string]any); ok {if v, ok := kwRaw["top_n"]; ok {p.Keywords.TopN = mapInt(v)}if v, ok := kwRaw["system_prompt"].(string); ok {p.Keywords.SystemPrompt = v- } else if v, ok := kwRaw["sys_prompt"].(string); ok {- p.Keywords.SystemPrompt = v- }- } else if v, ok := params["auto_keywords"]; ok {- p.Keywords.TopN = mapInt(v)- if v, ok := params["keywords_sys_prompt"].(string); ok {- p.Keywords.SystemPrompt = v}}- p.AutoKeywords = p.Keywords.TopN- // 2. Questions (modular or flat)+ // 2. Questionsif qRaw, ok := params["questions"].(map[string]any); ok {if v, ok := qRaw["top_n"]; ok {p.Questions.TopN = mapInt(v)}if v, ok := qRaw["system_prompt"].(string); ok {p.Questions.SystemPrompt = v- } else if v, ok := qRaw["sys_prompt"].(string); ok {- p.Questions.SystemPrompt = v- }- } else if v, ok := params["auto_questions"]; ok {- p.Questions.TopN = mapInt(v)- if v, ok := params["questions_sys_prompt"].(string); ok {- p.Questions.SystemPrompt = v}}- p.AutoQuestions = p.Questions.TopN- // 3. Tags (modular or flat)+ // 3. Tagsif tagRaw, ok := params["tags"].(map[string]any); ok {if v, ok := tagRaw["top_n"]; ok {p.Tags.TopN = mapInt(v)@@ -321,18 +241,9 @@ func NewExtractorComponent(params map[string]any) (runtime.Component, error) {if v, ok := tagRaw["tag_file_id"].(string); ok {p.Tags.TagFileID = v}- } else {- if v, ok := params["auto_tags"]; ok {- p.Tags.TopN = mapInt(v)- }- if v, ok := params["tag_file_id"].(string); ok {- p.Tags.TagFileID = v- }}- p.AutoTags = p.Tags.TopN- p.TagFileID = p.Tags.TagFileID- // 4. Summary (modular or flat)+ // 4. Summaryif sumRaw, ok := params["summary"].(map[string]any); ok {if v, ok := sumRaw["enabled"].(bool); ok {p.Summary.Enabled = v@@ -341,72 +252,22 @@ func NewExtractorComponent(params map[string]any) (runtime.Component, error) {}if v, ok := sumRaw["system_prompt"].(string); ok {p.Summary.SystemPrompt = v- } else if v, ok := sumRaw["sys_prompt"].(string); ok {- p.Summary.SystemPrompt = v- }- } else {- if v, ok := params["enable_summary"].(bool); ok {- p.Summary.Enabled = v- } else if v, ok := params["enable_summary"]; ok {- p.Summary.Enabled = mapInt(v) == 1- } else if p.FieldName == "summary" {- p.Summary.Enabled = true- }- p.Summary.SystemPrompt = p.SystemPrompt- }- if p.Summary.Enabled {- p.EnableSummary = 1- if p.FieldName == "" {- p.FieldName = "summary"- }- if p.Summary.SystemPrompt != "" && p.SystemPrompt == "" {- p.SystemPrompt = p.Summary.SystemPrompt- }- } else {- p.EnableSummary = 0- // When summary is explicitly disabled, avoid legacy "summary" field_name default triggering open-ended calls- if p.FieldName == "summary" {- p.FieldName = ""}}- // Metadata sub-config parsing with legacy flat fallback- if metaRaw, ok := params["metadata_config"].(map[string]any); ok {+ // 5. Metadata+ if metaRaw, ok := params["metadata"].(map[string]any); ok {if v, ok := metaRaw["enabled"].(bool); ok {- p.MetadataConfig.Enabled = v+ p.Metadata.Enabled = v} else if v, ok := metaRaw["enabled"]; ok {- p.MetadataConfig.Enabled = mapInt(v) == 1+ p.Metadata.Enabled = mapInt(v) == 1}if v, ok := metaRaw["metadata"]; ok {- p.MetadataConfig.Metadata = parseMetadataFieldDefs(v)+ p.Metadata.Metadata = parseMetadataFieldDefs(v)}if v, ok := metaRaw["built_in_metadata"]; ok {- p.MetadataConfig.BuiltInMetadata = parseMetadataFieldDefs(v)- }- if p.MetadataConfig.Enabled {- p.EnableMetadata = 1- } else {- p.EnableMetadata = 0- }- p.Metadata = p.MetadataConfig.Metadata- } else {- if v, ok := params["enable_metadata"].(bool); ok {- if v {- p.EnableMetadata = 1- } else {- p.EnableMetadata = 0- }- } else if v, ok := params["enable_metadata"]; ok {- p.EnableMetadata = mapInt(v)- }- if v, ok := params["metadata"]; ok {- p.Metadata = parseMetadataFieldDefs(v)- }- if v, ok := params["built_in_metadata"]; ok {- p.MetadataConfig.BuiltInMetadata = parseMetadataFieldDefs(v)+ p.Metadata.BuiltInMetadata = parseMetadataFieldDefs(v)}- p.MetadataConfig.Enabled = p.EnableMetadata > 0- p.MetadataConfig.Metadata = p.Metadata}}if err := p.Validate(); err != nil {@@ -421,21 +282,17 @@ func NewExtractorComponent(params map[string]any) (runtime.Component, error) {// self.chat_mdl; the Go port exposes it explicitly).func (c *ExtractorComponent) Inputs() map[string]string {return map[string]string{- "chunks": "List of map[string]any from upstream Tokenizer. Each entry must carry a string 'text' (or 'content_with_weight') field. Optional — when absent the LLM is called once with the resolved args.",- "prompt": "Optional user prompt template. Falls back to Param.Prompt when absent.",- "llm_id": "Optional per-call LLM id override. Falls back to Param.LLMID when absent.",- "system_prompt": "Optional per-call system prompt override. Falls back to Param.SystemPrompt.",+ "chunks": "List of map[string]any from upstream Tokenizer. Each entry must carry a string 'text' (or 'content_with_weight') field. Optional — when absent the LLM is called once with the resolved args.",+ "llm_id": "Optional per-call LLM id override. Falls back to Param.LLMID when absent.",}}// Outputs returns the public surface downstream ingestion// consumers can wire into. Mirrors schema.ExtractorOutputs.//-// chunks []map[string]any — input chunks, each augmented-// with field_name=<LLM result>.-// When the input chunks list is-// absent, the slice contains a-// single map with the same shape.+// chunks []map[string]any — input chunks, each augmented with+// extracted modular fields (important_kwd,+// question_kwd, tag_kwd, summary, metadata).// output_format string — always "chunks". Parity with// python set_output contract.// _ERROR string — populated on a short-circuit@@ -443,7 +300,7 @@ func (c *ExtractorComponent) Inputs() map[string]string {// set_output("_ERROR", ...)).func (c *ExtractorComponent) Outputs() map[string]string {return map[string]string{- "chunks": "Extraction results — input chunks (or a single-element slice when no chunks were supplied), each enriched with field_name=<LLM response>.",+ "chunks": "Extraction results — input chunks, each enriched with modular extraction fields (important_kwd, question_kwd, tag_kwd, summary, metadata).","output_format": "Always \"chunks\". Parity marker for downstream consumers.","_ERROR": "Optional short-circuit error message (reserved for the future TOC branch and other error paths).",}@@ -632,16 +489,12 @@ func toExtractorEinoMessages(msgs []eschema.Message) []*eschema.Message {// input map. Computed once at the top of Invoke so the rest of// the function reads as straight-line code.type extractorInputs struct {- fieldName string- llmID string- systemPrompt string- prompt string- lang string- chunks []map[string]any+ llmID string+ lang string+ chunks []map[string]any// temperature overrides the LLM temperature for this call. A// nil value leaves the request's Temperature unset so the model- // (or the chat-model default) decides, matching Python's generic- // Extractor path. The keyword/question helpers set it to+ // (or the chat-model default) decides. The keyword/question helpers set it to// extractorTemperature (0.2) to mirror generator.py.temperature *float64}@@ -649,16 +502,10 @@ type extractorInputs struct {// resolveInputs overlays per-call inputs on top of the// component's static Param. Missing keys fall back to the// Param-level values; per-call values win on conflict (so a-// canvas can override LLM_ID at runtime). The python-// Extractor reads inputs directly from get_input_elements(); the-// Go port normalizes to extractorInputs once at the top so the-// rest of Invoke reads straight-line.+// canvas can override LLM_ID at runtime).func (c *ExtractorComponent) resolveInputs(inputs map[string]any) extractorInputs {out := extractorInputs{- fieldName: c.Param.FieldName,- llmID: c.Param.LLMID,- systemPrompt: c.Param.SystemPrompt,- prompt: c.Param.Prompt,+ llmID: c.Param.LLMID,}if inputs == nil {return out@@ -666,14 +513,6 @@ func (c *ExtractorComponent) resolveInputs(inputs map[string]any) extractorInputif v, ok := inputs["llm_id"].(string); ok && v != "" {out.llmID = v}- if v, ok := inputs["prompt"].(string); ok && v != "" {- out.prompt = v- }- if v, ok := inputs["system_prompt"].(string); ok && v != "" {- out.systemPrompt = v- } else if v, ok := inputs["sys_prompt"].(string); ok && v != "" {- out.systemPrompt = v- }if v, ok := inputs["lang"].(string); ok && v != "" {out.lang = v}@@ -728,16 +567,11 @@ func extractorChunkList(v any) ([]map[string]any, bool) {//// chunks (optional, []map[string]any) — upstream chunks; each must// carry a string "text".-// prompt (optional, string) — overrides Param.Prompt.-// system_prompt (optional, string) — overrides Param.SystemPrompt.// llm_id (optional, string) — overrides Param.LLMID.//// Outputs://-// chunks ([]map[string]any) — input chunks augmented with-// field_name=<LLM result>. When-// the input list is empty, the-// slice contains a single map.+// chunks ([]map[string]any) — input chunks augmented with extraction results.// output_format (string) — always "chunks".// _ERROR (string, reserved) — populated when the component// short-circuits with an error.@@ -752,13 +586,16 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, db *gorm.DB, inputs mapzap.String("component", "Extractor"),zap.Int("input_chunks", len(in.chunks)),)- if in.fieldName == "toc" {- return nil, fmt.Errorf("extractor: field_name %q requires the TOC prompt generator which is not yet ported to Go", "toc")+ if len(in.chunks) == 0 {+ return map[string]any{+ "chunks": []map[string]any{},+ "output_format": "chunks",+ }, nil}if err := runtime.WithTimeout(ctx, extractorTimeout, func(timeoutCtx context.Context) error {- // Tag phase: run when auto_tags > 0 and we have chunks.- if c.Param.AutoTags > 0 && len(in.chunks) > 0 {+ // Tag phase: run when tags.top_n > 0 and we have chunks.+ if c.Param.Tags.TopN > 0 {tagged, tagErr := c.runAutoTags(timeoutCtx, db, in)if tagErr != nil {return tagErr@@ -766,62 +603,7 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, db *gorm.DB, inputs mapin.chunks = tagged}- if len(in.chunks) == 0 {- // Render the prompt with an empty chunk map so body placeholders- // resolve (or fall back to appending nothing) before the LLM call.- // Without this, a template containing {text} would be forwarded- // to the model unsubstituted.- callIn := in- if in.fieldName == "summary" && strings.TrimSpace(callIn.systemPrompt) == "" && strings.TrimSpace(c.Param.Summary.SystemPrompt) == "" {- callIn.systemPrompt = autoSummaryPrompt- }- callIn.systemPrompt, callIn.prompt = renderExtractorPrompts(- callIn.systemPrompt, in.prompt, map[string]any{}, "",- )- ans, callErr := c.callText(timeoutCtx, db, callIn, "")- if callErr != nil {- return callErr- }- ck := map[string]any{}- ck[in.fieldName] = ans- in.chunks = []map[string]any{ck}- return nil- }- // Auto keyword / question / metadata extraction runs with- // cross-chunk concurrency through the process-wide extractor- // pool (mirrors Python's chat_limiter). Each chunk job owns its- // own chunk map, so no per-chunk mutex is required; per-invocation- // completion is tracked inside runAutoExtractions.- if err := c.runAutoExtractions(timeoutCtx, db, in); err != nil {- return err- }-- // Primary field extraction stays sequential per chunk: each- // chunk's result lands under its own field_name key, preserving- // deterministic output ordering (see file header: -race tests).- for i, ck := range in.chunks {- if in.fieldName == "" {- continue- }- text, _ := ck["content_with_weight"].(string)- if strings.TrimSpace(text) == "" {- text, _ = ck["text"].(string)- }- // Substitute {field_name} placeholders with current- // chunk field values, mirroring Python's- // string_format at extractor.py:103.- callIn := in- if in.fieldName == "summary" && strings.TrimSpace(callIn.systemPrompt) == "" && strings.TrimSpace(c.Param.Summary.SystemPrompt) == "" {- callIn.systemPrompt = autoSummaryPrompt- }- callIn.systemPrompt, callIn.prompt = renderExtractorPrompts(callIn.systemPrompt, in.prompt, ck, text)- ans, callErr := c.callText(timeoutCtx, db, callIn, "")- if callErr != nil {- return fmt.Errorf("chunk %d: %w", i, callErr)- }- ck[in.fieldName] = ans- }- return nil+ return c.runAutoExtractions(timeoutCtx, db, in)}); err != nil {return nil, fmt.Errorf("extractor: %w", err)}@@ -836,35 +618,26 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map}// runAutoKeywords extracts keywords for the current chunk and stores-// them on ck["important_kwd"]. mu may be nil (sequential path); when-// non-nil it serializes the shared chunk-map accesses so concurrent-// keyword/question goroutines stay race-free. The existence check and-// the map writes are both guarded by mu. Keyword extraction pins+// them on ck["important_kwd"]. Keyword extraction pins// temperature to extractorTemperature (0.2) to mirror generator.py.func (c *ExtractorComponent) runAutoKeywords(ctx context.Context, db *gorm.DB, in extractorInputs, ck map[string]any, chunkText string) error {- _, exists := ck["important_kwd"]- if exists {+ if _, exists := ck["important_kwd"]; exists {return nil}topN := c.Param.Keywords.TopNif topN <= 0 {- topN = c.Param.AutoKeywords+ return nil}- var sysPrompt, userPrompt string- if strings.TrimSpace(c.Param.Keywords.SystemPrompt) != "" {- sysPrompt, userPrompt = renderExtractorPrompts(c.Param.Keywords.SystemPrompt, "Output: ", ck, chunkText)- } else {- sysPrompt = fmt.Sprintf(autoKeywordPrompt, topN, chunkText)- userPrompt = "Output: "+ systemPrompt := c.Param.Keywords.SystemPrompt+ if strings.TrimSpace(systemPrompt) == "" {+ systemPrompt = fmt.Sprintf(autoKeywordPrompt, topN)}kwTemp := extractorTemperaturekwIn := extractorInputs{- llmID: in.llmID,- systemPrompt: sysPrompt,- prompt: userPrompt,- temperature: &kwTemp,+ llmID: in.llmID,+ temperature: &kwTemp,}- resultStr, err := c.callText(ctx, db, kwIn, "")+ resultStr, err := c.callText(ctx, db, kwIn, systemPrompt, chunkText)if err != nil {return err}@@ -888,29 +661,23 @@ func (c *ExtractorComponent) runAutoKeywords(ctx context.Context, db *gorm.DB, i// runAutoQuestions extracts questions for the current chunk and stores// them on ck["question_kwd"]. See runAutoKeywords for the temperature pin.func (c *ExtractorComponent) runAutoQuestions(ctx context.Context, db *gorm.DB, in extractorInputs, ck map[string]any, chunkText string) error {- _, exists := ck["question_kwd"]- if exists {+ if _, exists := ck["question_kwd"]; exists {return nil}topN := c.Param.Questions.TopNif topN <= 0 {- topN = c.Param.AutoQuestions+ return nil}- var sysPrompt, userPrompt string- if strings.TrimSpace(c.Param.Questions.SystemPrompt) != "" {- sysPrompt, userPrompt = renderExtractorPrompts(c.Param.Questions.SystemPrompt, "Output: ", ck, chunkText)- } else {- sysPrompt = fmt.Sprintf(autoQuestionPrompt, topN, chunkText)- userPrompt = "Output: "+ systemPrompt := c.Param.Questions.SystemPrompt+ if strings.TrimSpace(systemPrompt) == "" {+ systemPrompt = fmt.Sprintf(autoQuestionPrompt, topN)}qTemp := extractorTemperatureqIn := extractorInputs{- llmID: in.llmID,- systemPrompt: sysPrompt,- prompt: userPrompt,- temperature: &qTemp,+ llmID: in.llmID,+ temperature: &qTemp,}- resultStr, err := c.callText(ctx, db, qIn, "")+ resultStr, err := c.callText(ctx, db, qIn, systemPrompt, chunkText)if err != nil {return err}@@ -939,7 +706,37 @@ func (c *ExtractorComponent) runAutoQuestions(ctx context.Context, db *gorm.DB,return nil}-// runAutoExtractions dispatches the auto keyword / question / metadata+// runAutoSummary extracts a concise summary for the current chunk using autoSummaryPrompt+// and stores it on ck["summary"].+func (c *ExtractorComponent) runAutoSummary(ctx context.Context, db *gorm.DB, in extractorInputs, ck map[string]any, chunkText string) error {+ if _, exists := ck["summary"]; exists {+ return nil+ }+ if !c.Param.Summary.Enabled {+ return nil+ }+ systemPrompt := c.Param.Summary.SystemPrompt+ if strings.TrimSpace(systemPrompt) == "" {+ systemPrompt = autoSummaryPrompt+ }+ sumTemp := extractorTemperature+ sumIn := extractorInputs{+ llmID: in.llmID,+ temperature: &sumTemp,+ }+ resultStr, err := c.callText(ctx, db, sumIn, systemPrompt, chunkText)+ if err != nil {+ return err+ }+ resultStr = cleanExtractionResult(resultStr)+ if resultStr == "" {+ return nil+ }+ ck["summary"] = resultStr+ return nil+}++// runAutoExtractions dispatches the auto keyword / question / summary / metadata// extraction for every chunk to the process-wide extractorPool so the// work runs with bounded cross-chunk concurrency (mirrors Python's// chat_limiter). The pool only bounds concurrency; per-invocation@@ -947,7 +744,7 @@ func (c *ExtractorComponent) runAutoQuestions(ctx context.Context, db *gorm.DB,// sync.WaitGroup, so concurrent Invoke calls do not disturb each other.// Each chunk job owns its own chunk map, so no per-chunk mutex is needed.func (c *ExtractorComponent) runAutoExtractions(ctx context.Context, db *gorm.DB, in extractorInputs) error {- if c.Param.AutoKeywords == 0 && c.Param.AutoQuestions == 0 && c.Param.EnableMetadata == 0 {+ if c.Param.Keywords.TopN <= 0 && c.Param.Questions.TopN <= 0 && !c.Param.Summary.Enabled && !c.Param.Metadata.Enabled {return nil}futs := make([]utility.WorkerPoolFuture[extractorJob, struct{}], 0, len(in.chunks))@@ -994,23 +791,28 @@ func (c *ExtractorComponent) runAutoExtractions(ctx context.Context, db *gorm.DB}// autoExtractionJob returns the unit of work for one chunk: it runs the-// enabled auto-extractions (keywords, questions, metadata) sequentially+// enabled auto-extractions (keywords, questions, summary, metadata) sequentially// on the chunk's own map. Running them sequentially inside the job keeps// the code free of per-chunk mutexes — cross-chunk parallelism is provided// by the pool, not by intra-chunk goroutines.func (c *ExtractorComponent) autoExtractionJob(ctx context.Context, db *gorm.DB, in extractorInputs, idx int, ck map[string]any, chunkText string) extractorJob {return func() error {- if c.Param.AutoKeywords > 0 {+ if c.Param.Keywords.TopN > 0 {if err := c.runAutoKeywords(ctx, db, in, ck, chunkText); err != nil {return fmt.Errorf("chunk %d keywords: %w", idx, err)}}- if c.Param.AutoQuestions > 0 {+ if c.Param.Questions.TopN > 0 {if err := c.runAutoQuestions(ctx, db, in, ck, chunkText); err != nil {return fmt.Errorf("chunk %d questions: %w", idx, err)}}- if c.Param.EnableMetadata > 0 {+ if c.Param.Summary.Enabled {+ if err := c.runAutoSummary(ctx, db, in, ck, chunkText); err != nil {+ return fmt.Errorf("chunk %d summary: %w", idx, err)+ }+ }+ if c.Param.Metadata.Enabled {if err := c.runEnableMetadata(ctx, db, in, ck, chunkText); err != nil {return fmt.Errorf("chunk %d metadata: %w", idx, err)}@@ -1022,15 +824,14 @@ func (c *ExtractorComponent) autoExtractionJob(ctx context.Context, db *gorm.DB,// runEnableMetadata extracts structured metadata for the current chunk and// merges the parsed JSON object into ck["metadata"]. It mirrors the// runAutoKeywords/runAutoQuestions shape but parses a JSON object and-// merges into the chunk's metadata map (which the existing-// mergeChunkMetadata → docState.apply aggregation path consumes).+// merges into the chunk's metadata map.func (c *ExtractorComponent) runEnableMetadata(ctx context.Context, db *gorm.DB, in extractorInputs, ck map[string]any, chunkText string) error {- if len(c.Param.Metadata) == 0 {+ if !c.Param.Metadata.Enabled || len(c.Param.Metadata.Metadata) == 0 {return nil}// Render the field schema into the prompt, mirroring Python's// turn2jsonschema(metadata_conf) rendered into the META_DATA template.- schemaMap := common.Turn2JSONSchema(c.Param.Metadata)+ schemaMap := common.Turn2JSONSchema(c.Param.Metadata.Metadata)if len(schemaMap) == 0 {return nil}@@ -1052,19 +853,16 @@ func (c *ExtractorComponent) runEnableMetadata(ctx context.Context, db *gorm.DB,} else {metaTemp := extractorTemperaturemetaIn := extractorInputs{- llmID: in.llmID,- systemPrompt: fmt.Sprintf(autoMetadataPrompt, schemaStr, chunkText),- prompt: "Output: ",- temperature: &metaTemp,+ llmID: in.llmID,+ temperature: &metaTemp,}- parsed, err = c.callStructured(ctx, db, metaIn, "")+ systemPrompt := fmt.Sprintf(autoMetadataPrompt, schemaStr)+ parsed, err = c.callStructured(ctx, db, metaIn, systemPrompt, chunkText)if err != nil {return err}if parsed == nil {// Non-JSON or empty response — nothing to extract, not an error.- // Matches Python gen_metadata treating an unparseable reply as an- // empty result rather than failing the chunk.return nil}setMetadataLLMCache(ctx, in.llmID, schemaStr, chunkText, parsed)@@ -1138,6 +936,58 @@ func setMetadataLLMCache(ctx context.Context, llmID, schemaJSON, chunkText strinclient.Set(ctx, metadataLLMCacheKey(llmID, schemaJSON, chunkText), string(data), metadataLLMCacheTTL)}+// callRaw runs one chat call against the LLM (per chunk in the normal path)+// and returns the raw response. It holds the shared plumbing — driver/target+// resolution, message assembly, temperature override, retry — but deliberately+// does NOT clean or parse the response. Callers pick the view they need:+//+// - callText wraps callRaw with the LLM-layer two-step cleanup (think ++// tool_call) and returns a plain string.+// - callStructured wraps callRaw with cleanup + explicit JSON parsing and+// returns a map — the metadata path, matching Python gen_metadata.+func (c *ExtractorComponent) callRaw(ctx context.Context, db *gorm.DB, in extractorInputs, systemPrompt, chunkText string) (*extractorChatResponse, error) {+ driver, modelName, apiKey, baseURL, err := resolveExtractorChatTarget(ctx, db, in.llmID)+ if err != nil {+ return nil, err+ }+ msgs := buildExtractorMessages(systemPrompt, chunkText)+ fitted, fitErr := fitExtractorMessages(ctx, db, in.llmID, msgs)+ if fitErr != nil {+ return nil, fitErr+ }+ msgs = fitted+ inv := getExtractorChatInvoker()+ req := extractorChatRequest{+ Driver: driver,+ ModelName: modelName,+ APIKey: apiKey,+ BaseURL: baseURL,+ Messages: msgs,+ }+ // Only override the temperature when the caller set one. A nil+ // Temperature lets the model / chat-model default decide, matching+ // Python's generic Extractor path; keyword/question helpers set+ // extractorTemperature (0.2) to mirror generator.py.+ if in.temperature != nil {+ temp := *in.temperature+ req.Temperature = &temp+ }+ var resp *extractorChatResponse+ if err := common.RetryWithBackoff(ctx, extractorRetryMax, extractorRetryDelay, func() error {+ r, e := inv.Chat(ctx, req)+ resp = r+ return e+ }, isRetryableLLMError); err != nil {+ return nil, err+ }+ if resp == nil {+ // Defensive: a provider adapter returning (nil, nil) would otherwise+ // panic on resp.Content below. Surface a diagnosable error instead.+ return nil, fmt.Errorf("extractor: chat: nil response from invoker")+ }+ return resp, nil+}+// toolCallRE matches a <tool_call>...</tool_call> block, mirroring Python's// `re.sub(r"<tool_call>.*?</tool_call>", "", txt, flags=re.DOTALL)` at// llm_service.py:461. Non-greedy so consecutive tool_call blocks are each@@ -1236,72 +1086,9 @@ func isRetryableLLMError(err error) bool {return true}-// callRaw dispatches one LLM chat call for the supplied chunk text (empty-// string in the no-chunk fast path) and returns the raw response. It holds-// the shared plumbing — driver/target resolution, message assembly,-// temperature override, retry — but deliberately does NOT clean or parse the-// response. Callers pick the view they need:-//-// - callText wraps callRaw with the LLM-layer two-step cleanup (think +-// tool_call) and returns a plain string — the field-extraction path,-// matching Python _generate_async.-// - callStructured wraps callRaw with cleanup + explicit JSON parsing and-// returns a map — the metadata path, matching Python gen_metadata.-//-// Splitting this way restores Python's separation of concerns: the LLM layer-// (async_chat) cleans, the caller decides whether to parse. A single-// all-purpose call() that best-effort parses would hand field extraction a-// map it does not want (silently dropped downstream) while leaving metadata-// to re-parse — and could not add cleaning without breaking JSON structure.-func (c *ExtractorComponent) callRaw(ctx context.Context, db *gorm.DB, in extractorInputs, chunkText string) (*extractorChatResponse, error) {- driver, modelName, apiKey, baseURL, err := resolveExtractorChatTarget(ctx, db, in.llmID)- if err != nil {- return nil, err- }- msgs := buildExtractorMessages(in.systemPrompt, in.prompt)- fitted, fitErr := fitExtractorMessages(ctx, db, in.llmID, msgs)- if fitErr != nil {- return nil, fitErr- }- msgs = fitted- inv := getExtractorChatInvoker()- req := extractorChatRequest{- Driver: driver,- ModelName: modelName,- APIKey: apiKey,- BaseURL: baseURL,- Messages: msgs,- }- // Only override the temperature when the caller set one. A nil- // Temperature lets the model / chat-model default decide, matching- // Python's generic Extractor path; keyword/question helpers set- // extractorTemperature (0.2) to mirror generator.py.- if in.temperature != nil {- temp := *in.temperature- req.Temperature = &temp- }- var resp *extractorChatResponse- if err := common.RetryWithBackoff(ctx, extractorRetryMax, extractorRetryDelay, func() error {- r, e := inv.Chat(ctx, req)- resp = r- return e- }, isRetryableLLMError); err != nil {- return nil, err- }- if resp == nil {- // Defensive: a provider adapter returning (nil, nil) would otherwise- // panic on resp.Content below. Surface a diagnosable error instead.- return nil, fmt.Errorf("extractor: chat: nil response from invoker")- }- return resp, nil-}--// callText runs one chat call and returns the cleaned text response. Used by-// the field-extraction path (empty-chunks fast path and per-chunk writes),-// which must always produce a string — matching Python's _generate_async-// (llm.py:374-377) returning a raw string with no JSON parsing.-func (c *ExtractorComponent) callText(ctx context.Context, db *gorm.DB, in extractorInputs, chunkText string) (string, error) {- resp, err := c.callRaw(ctx, db, in, chunkText)+// callText runs one chat call and returns the cleaned text response.+func (c *ExtractorComponent) callText(ctx context.Context, db *gorm.DB, in extractorInputs, systemPrompt, chunkText string) (string, error) {+ resp, err := c.callRaw(ctx, db, in, systemPrompt, chunkText)if err != nil {return "", err}@@ -1313,8 +1100,8 @@ func (c *ExtractorComponent) callText(ctx context.Context, db *gorm.DB, in extra// structured value to merge into chunk metadata — matching Python// gen_metadata's json_repair.loads. A non-JSON or empty response returns// (nil, nil) so the caller treats it as "nothing extracted", not an error.-func (c *ExtractorComponent) callStructured(ctx context.Context, db *gorm.DB, in extractorInputs, chunkText string) (map[string]any, error) {- resp, err := c.callRaw(ctx, db, in, chunkText)+func (c *ExtractorComponent) callStructured(ctx context.Context, db *gorm.DB, in extractorInputs, systemPrompt, chunkText string) (map[string]any, error) {+ resp, err := c.callRaw(ctx, db, in, systemPrompt, chunkText)if err != nil {return nil, err}@@ -1644,107 +1431,21 @@ func fitExtractorMessages(ctx context.Context, db *gorm.DB, llmID string, msgs [}// buildExtractorMessages assembles system + user messages for one extraction-// call. Prompt rendering (placeholder substitution, chunk-text injection, and-// empty-prompt normalization) is performed upstream by renderExtractorPrompts;-// this function is a pure structural assembler with no rendering logic.-func buildExtractorMessages(system, user string) []eschema.Message {+// call. The user message strictly carries chunkText (or a fallback single space+// if empty), ensuring a clean and consistent message contract.+// System prompt is omitted if empty or whitespace-only to avoid sending empty+// system turns to LLM providers.+func buildExtractorMessages(systemPrompt, chunkText string) []eschema.Message {out := make([]eschema.Message, 0, 2)- if system != "" {- out = append(out, eschema.Message{Role: eschema.System, Content: system})- }- out = append(out, eschema.Message{Role: eschema.User, Content: user})- return out-}--// unifiedPlaceholderRE matches plain field placeholders {fieldName} and-// canvas macro placeholders {ComponentName:ParamName@fieldName}.-var unifiedPlaceholderRE = regexp.MustCompile(`\{([A-Za-z0-9_]+(:[A-Za-z0-9_]+)?@[A-Za-z0-9_]+|[A-Za-z_][A-Za-z0-9_]*)\}`)--// bodyPlaceholderAliases lists plain placeholder names that resolve to chunk body content.-var bodyPlaceholderAliases = map[string]bool{- "text": true,- "chunks": true,- "content_with_weight": true,-}--// isBodyPlaceholder reports whether key represents chunk body content-// (including upstream output references such as @chunks, @text, @markdown).-func isBodyPlaceholder(key string) bool {- if bodyPlaceholderAliases[key] {- return true- }- return strings.HasSuffix(key, "@chunks") ||- strings.HasSuffix(key, "@text") ||- strings.HasSuffix(key, "@markdown")-}--// renderExtractorPrompts performs single-pass rendering of system and user-// prompt templates for one chunk:-// 1. Content placeholders ({text}, {chunks}, {content_with_weight},-// {ComponentName:ParamName@chunks}, etc.) are resolved: first from the chunk-// map, falling back to chunkText. If a non-empty value is resolved,-// bodyInjected is marked true.-// 2. Chunk metadata fields (such as {title}, {author}) are resolved from the-// chunk map.-// 3. Unrecognized placeholders are preserved verbatim.-// 4. If no non-empty chunk body was injected anywhere in the system or user-// prompt, chunkText is automatically appended to the user prompt as a fallback.-func renderExtractorPrompts(sysTemplate, userTemplate string, ck map[string]any, chunkText string) (string, string) {- var bodyInjected bool-- render := func(tmpl string) string {- if tmpl == "" {- return ""- }- return unifiedPlaceholderRE.ReplaceAllStringFunc(tmpl, func(match string) string {- key := match[1 : len(match)-1] // strip { }-- // A. Content placeholders: check chunk map first, fallback to chunkText- if isBodyPlaceholder(key) {- var val string- if raw, ok := ck[key]; ok {- val = fmt.Sprintf("%v", raw)- }- if strings.TrimSpace(val) == "" {- val = chunkText- }- if strings.TrimSpace(val) != "" {- bodyInjected = true- }- return val- }-- // B. Chunk metadata fields- if val, ok := ck[key]; ok {- return fmt.Sprintf("%v", val)- }-- // C. Leave unknown placeholders as-is- return match- })- }-- renderedSys := render(sysTemplate)- renderedUser := render(userTemplate)-- if !bodyInjected && strings.TrimSpace(chunkText) != "" {- // TrimSpace before testing emptiness is intentional: a user template- // that renders to pure whitespace (e.g. " ") must not produce a- // leading "\n\n" separator — the result should be chunkText alone.- // A plain `if renderedUser != ""` check would fail that invariant.- trimmed := strings.TrimSpace(renderedUser)- if trimmed != "" {- renderedUser = trimmed + "\n\n" + chunkText- } else {- renderedUser = chunkText- }+ if strings.TrimSpace(systemPrompt) != "" {+ out = append(out, eschema.Message{Role: eschema.System, Content: systemPrompt})}-- if strings.TrimSpace(renderedUser) == "" {- renderedUser = " "+ userContent := chunkText+ if strings.TrimSpace(userContent) == "" {+ userContent = " "}-- return renderedSys, renderedUser+ out = append(out, eschema.Message{Role: eschema.User, Content: userContent})+ return out}// tryParseJSONObject tries to parse s as a JSON object. Returns@@ -1801,8 +1502,22 @@ func tryParseJSONObject(s string) (map[string]any, bool) {// parseMetadataFieldDefs converts an any value (typically []any of maps)// to a typed []common.MetadataFieldDef slice.func parseMetadataFieldDefs(v any) []common.MetadataFieldDef {- arr, ok := v.([]any)- if !ok {+ if v == nil {+ return nil+ }+ if defs, ok := v.([]common.MetadataFieldDef); ok {+ return defs+ }+ var arr []any+ switch typed := v.(type) {+ case []any:+ arr = typed+ case []map[string]any:+ arr = make([]any, 0, len(typed))+ for _, item := range typed {+ arr = append(arr, item)+ }+ default:return nil}fields := make([]common.MetadataFieldDef, 0, len(arr))diff --git a/internal/ingestion/component/extractor_tag.go b/internal/ingestion/component/extractor_tag.goindex a84b759a41..ec82e16a69 100644--- a/internal/ingestion/component/extractor_tag.go+++ b/internal/ingestion/component/extractor_tag.go@@ -155,7 +155,7 @@ func (c *ExtractorComponent) runAutoTags(ctx context.Context, db *gorm.DB, in exreturn in.chunks, nil}- topN := c.Param.AutoTags+ topN := c.Param.Tags.TopNvar examples []schema.TaggedChunkvar docsToTag []map[string]any@@ -218,22 +218,22 @@ func (c *ExtractorComponent) runAutoTags(ctx context.Context, db *gorm.DB, in ex}func (c *ExtractorComponent) resolveTagSource(ctx context.Context) (*indexedTagSource, bool) {- if c.Param.TagFileID == "" {+ if c.Param.Tags.TagFileID == "" {return nil, false}return c.loadTagFileIndexed(ctx)}func (c *ExtractorComponent) loadTagFileIndexed(ctx context.Context) (*indexedTagSource, bool) {- f, err := dao.NewFileDAO().GetByID(ctx, dao.DB, c.Param.TagFileID)+ f, err := dao.NewFileDAO().GetByID(ctx, dao.DB, c.Param.Tags.TagFileID)if err != nil || f == nil || f.Location == nil || *f.Location == "" {- common.Warn(fmt.Sprintf("extractor tags: resolve tag_file_id %q: %v", c.Param.TagFileID, err))+ common.Warn(fmt.Sprintf("extractor tags: resolve tag_file_id %q: %v", c.Param.Tags.TagFileID, err))return nil, false}cacheKey := tagSourceFileCacheKey(f)if cached, ok := tagSourceFileIndexCache.load(cacheKey); ok {common.Info("extractor tags: reused tag source file index",- zap.String("file_id", c.Param.TagFileID),+ zap.String("file_id", c.Param.Tags.TagFileID),zap.String("bucket", f.ParentID),zap.String("key", *f.Location),)@@ -256,7 +256,7 @@ func (c *ExtractorComponent) loadTagFileIndexed(ctx context.Context) (*indexedTa}indexed = tagSourceFileIndexCache.store(cacheKey, indexed)common.Info("extractor tags: loaded tag source file",- zap.String("file_id", c.Param.TagFileID),+ zap.String("file_id", c.Param.Tags.TagFileID),zap.String("bucket", f.ParentID),zap.String("key", *f.Location),zap.Int64("size", f.Size),diff --git a/internal/ingestion/component/extractor_tag_test.go b/internal/ingestion/component/extractor_tag_test.goindex 9e7c8df374..a5260bb77e 100644--- a/internal/ingestion/component/extractor_tag_test.go+++ b/internal/ingestion/component/extractor_tag_test.go@@ -45,7 +45,7 @@ func pushExtractorTagTargetResolverStub(t *testing.T) {func TestExtractorTags_NoTagFileID(t *testing.T) {pushExtractorTagChatStub(t, nil)- comp, _ := NewExtractorComponent(map[string]any{"auto_tags": 3})+ comp, _ := NewExtractorComponent(map[string]any{"tags": map[string]any{"top_n": 3}})out, err := comp.Invoke(t.Context(), nil, map[string]any{"chunks": []map[string]any{{"content_with_weight": "test"},@@ -65,7 +65,7 @@ func TestExtractorTags_NoTagFileID(t *testing.T) {func TestExtractorTags_NoLLMID(t *testing.T) {pushExtractorTagChatStub(t, nil)- comp, _ := NewExtractorComponent(map[string]any{"auto_tags": 3})+ comp, _ := NewExtractorComponent(map[string]any{"tags": map[string]any{"top_n": 3}})out, err := comp.Invoke(t.Context(), nil, map[string]any{"chunks": []map[string]any{{"content_with_weight": "some unrelated text"},@@ -85,9 +85,13 @@ func TestExtractorTags_WithKeywords(t *testing.T) {pushExtractorTagTargetResolverStub(t)comp, _ := NewExtractorComponent(map[string]any{- "llm_id": "test@test",- "auto_tags": 3,- "auto_keywords": 3,+ "llm_id": "test@test",+ "tags": map[string]any{+ "top_n": 3,+ },+ "keywords": map[string]any{+ "top_n": 3,+ },})out, err := comp.Invoke(t.Context(), nil, map[string]any{"chunks": []map[string]any{diff --git a/internal/ingestion/component/extractor_test.go b/internal/ingestion/component/extractor_test.goindex fd738a0a0e..e5aff9f85b 100644--- a/internal/ingestion/component/extractor_test.go+++ b/internal/ingestion/component/extractor_test.go@@ -102,8 +102,7 @@ func withStubChatInvoker(t *testing.T, responses ...stubResponse) *stubExtractor}// TestExtractorComponent_Registered verifies the init() registration-// is visible to the runtime registry (Phase 4 / API layer-// depends on this).+// is visible to the runtime registry.func TestExtractorComponent_Registered(t *testing.T) {factory, cat, md, ok := runtime.DefaultRegistry.Lookup("Extractor")if !ok {@@ -130,8 +129,7 @@ func TestExtractorComponent_Registered(t *testing.T) {}// TestExtractorComponent_Invoke_HappyPath covers the per-chunk-// fan-out: two chunks in → two LLM calls → each chunk enriched-// with the field_name key.+// auto-extraction (e.g. summary).func TestExtractorComponent_Invoke_HappyPath(t *testing.T) {withStubChatInvoker(t,stubResponse{Content: "answer for chunk 1"},@@ -139,9 +137,8 @@ func TestExtractorComponent_Invoke_HappyPath(t *testing.T) {)c := &ExtractorComponent{Param: schema.ExtractorParam{- FieldName: "summary",- LLMID: "gpt-4o-mini",- Prompt: "Summarize:",+ LLMID: "gpt-4o-mini",+ Summary: schema.SummaryExtractConfig{Enabled: true},}}out, err := c.Invoke(t.Context(), nil, map[string]any{"chunks": []map[string]any{@@ -160,11 +157,10 @@ func TestExtractorComponent_Invoke_HappyPath(t *testing.T) {if len(chunks) != 2 {t.Fatalf("chunks len = %d, want 2", len(chunks))}- if chunks[0]["summary"] != "answer for chunk 1" {- t.Errorf("chunk[0].summary = %v, want %q", chunks[0]["summary"], "answer for chunk 1")- }- if chunks[1]["summary"] != "answer for chunk 2" {- t.Errorf("chunk[1].summary = %v, want %q", chunks[1]["summary"], "answer for chunk 2")+ s0, _ := chunks[0]["summary"].(string)+ s1, _ := chunks[1]["summary"].(string)+ if !((s0 == "answer for chunk 1" && s1 == "answer for chunk 2") || (s0 == "answer for chunk 2" && s1 == "answer for chunk 1")) {+ t.Errorf("unexpected summaries: chunk0=%q, chunk1=%q", s0, s1)}if out["output_format"] != "chunks" {t.Errorf("output_format = %v, want chunks", out["output_format"])@@ -172,11 +168,8 @@ func TestExtractorComponent_Invoke_HappyPath(t *testing.T) {}// TestExtractorComponent_Invoke_LLMError verifies a mock LLM-// error is surfaced through Invoke with the component-name prefix-// so the upstream pipeline can attribute failures. After retry-// (RetryWithBackoff: 3 retries), the error chains the cause.+// error is surfaced through Invoke with the component-name prefix.func TestExtractorComponent_Invoke_LLMError(t *testing.T) {- // Fast retry for tests — avoid multi-second sleeps.prevMax, prevDelay := extractorRetryMax, extractorRetryDelayextractorRetryMax, extractorRetryDelay = 3, time.Millisecondt.Cleanup(func() {@@ -192,8 +185,8 @@ func TestExtractorComponent_Invoke_LLMError(t *testing.T) {)c := &ExtractorComponent{Param: schema.ExtractorParam{- FieldName: "summary",- LLMID: "gpt-4o-mini",+ LLMID: "gpt-4o-mini",+ Summary: schema.SummaryExtractConfig{Enabled: true},}}_, err := c.Invoke(t.Context(), nil, map[string]any{"chunks": []map[string]any{{"text": "x"}},@@ -226,8 +219,8 @@ func TestExtractorComponent_Invoke_RetrySucceeds(t *testing.T) {)c := &ExtractorComponent{Param: schema.ExtractorParam{- FieldName: "summary",- LLMID: "gpt-4o-mini",+ LLMID: "gpt-4o-mini",+ Summary: schema.SummaryExtractConfig{Enabled: true},}}out, err := c.Invoke(t.Context(), nil, map[string]any{"chunks": []map[string]any{{"text": "x"}},@@ -246,49 +239,17 @@ func TestExtractorComponent_Invoke_RetrySucceeds(t *testing.T) {// TestExtractorComponent_Invoke_UnknownProvider asserts the// production (eino) chat invoker handles an unregistered driver-// without panicking, per plan §8 Q1 ("48/56 providers covered;-// the Extractor is provider-agnostic via llm_id; the 8 missing-// are edge cases that do not block Phase 2.5").-//-// Design note: every other test in this file drives the-// invoker through the production Component.Invoke path with a-// canned-response invoker installed via SetExtractorChatInvoker-// (the test seam). That seam accepts a pre-resolved driver-// path; it cannot model the eino factory's default-branch-// behaviour for an unknown driver. This test exercises the-// production chat-invoker directly to pin that branch — the-// production code path the real Extractor will hit when the-// DSL references a provider that is not in the 48/56 covered-// set.-//-// The contract under test:-// - The call MUST NOT panic.-// - On unknown driver, the factory's default branch routes to-// a DummyModel that returns a deterministic error string-// (we assert the error contains that sentinel so future-// maintainers see the wiring goes through the factory,-// not bypassed by a hand-rolled default).+// without panicking.func TestExtractorComponent_Invoke_UnknownProvider(t *testing.T) {inv := &einoExtractorChatInvoker{}resp, err := inv.Chat(context.Background(), extractorChatRequest{Driver: "definitely-not-a-real-provider-xyz",ModelName: "anything",})- // Either an error is returned OR a non-nil response is produced- // by the DummyModel fallback. The contract is "no panic"; both- // of these outcomes are acceptable. We only fail the test if- // BOTH error and response are empty (which would indicate a- // silent no-op).if err == nil && resp == nil {t.Fatal("production invoker returned nil error AND nil response for unknown driver — silent no-op")}- // When an error IS returned, it must mention the driver name so- // operators can correlate the failure back to the DSL config.if err != nil {- // Acceptable error patterns for an unknown driver:- // - mentions the driver name (correlatable for operators)- // - "no driver"/"unknown" sentinels (typed error)- // - "not implemented" (the eino dummy model fallback path)if !strings.Contains(err.Error(), "definitely-not-a-real-provider-xyz") &&!strings.Contains(err.Error(), "no driver") &&!strings.Contains(err.Error(), "unknown") &&@@ -298,120 +259,11 @@ func TestExtractorComponent_Invoke_UnknownProvider(t *testing.T) {}}-// TestExtractorComponent_Invoke_KeepsJSONAsString verifies a JSON object-// response from the LLM is written to the chunk's field_name value as a-// plain string — matching Python's _generate_async, which returns the raw-// string with no JSON parsing. (The Extractor does NOT parse field-extraction-// results; only the metadata path, via callStructured, parses explicitly.)-func TestExtractorComponent_Invoke_KeepsJSONAsString(t *testing.T) {- withStubChatInvoker(t,- stubResponse{Content: `{"answer": 42, "tags": ["a", "b"]}`},- )-- c := &ExtractorComponent{Param: schema.ExtractorParam{- FieldName: "extraction",- Prompt: "extract:",- }}- out, err := c.Invoke(t.Context(), nil, map[string]any{- "chunks": []map[string]any{{"text": "doc"}}},- )- if err != nil {- t.Fatalf("Invoke: %v", err)- }- chunks := out["chunks"].([]map[string]any)- got, ok := chunks[0]["extraction"].(string)- if !ok {- t.Fatalf("extraction should be a string (no JSON parse on field extraction), got %T", chunks[0]["extraction"])- }- if got != `{"answer": 42, "tags": ["a", "b"]}` {- t.Errorf("extraction = %q, want the raw JSON string", got)- }-}--// TestExtractorComponent_Invoke_KeepsJSONStringInFence verifies a JSON-// response wrapped in a Markdown code fence is stored as the raw string —-// the code fence is not stripped on the field-extraction path (Python's-// _generate_async returns the raw text untouched).-func TestExtractorComponent_Invoke_KeepsJSONStringInFence(t *testing.T) {- withStubChatInvoker(t,- stubResponse{Content: "```json\n{\"summary\": \"hello\"}\n```"},- )-- c := &ExtractorComponent{Param: schema.ExtractorParam{- FieldName: "out",- }}- out, err := c.Invoke(t.Context(), nil, map[string]any{- "chunks": []map[string]any{{"text": "x"}}},- )- if err != nil {- t.Fatalf("Invoke: %v", err)- }- got, ok := out["chunks"].([]map[string]any)[0]["out"].(string)- if !ok {- t.Fatalf("out should be a string, got %T", out["chunks"].([]map[string]any)[0]["out"])- }- if got != "```json\n{\"summary\": \"hello\"}\n```" {- t.Errorf("out = %q, want the raw fenced JSON string", got)- }-}--// TestExtractorComponent_Invoke_HandlesMalformedJSON verifies a-// non-JSON response surfaces as the raw string under the-// destination field — not an error. The python Extractor-// accepts whatever the LLM emits; downstream callers decide-// what to do with it.-func TestExtractorComponent_Invoke_HandlesMalformedJSON(t *testing.T) {- withStubChatInvoker(t,- stubResponse{Content: "this is not JSON at all"},- )-- c := &ExtractorComponent{Param: schema.ExtractorParam{- FieldName: "raw",- }}- out, err := c.Invoke(t.Context(), nil, map[string]any{- "chunks": []map[string]any{{"text": "x"}}},- )- if err != nil {- t.Fatalf("Invoke returned error on non-JSON: %v", err)- }- got := out["chunks"].([]map[string]any)[0]["raw"]- if got != "this is not JSON at all" {- t.Errorf("raw = %v, want %q", got, "this is not JSON at all")- }-}--// TestExtractorComponent_Invoke_TOCNotPorted asserts the-// field_name=="toc" branch is gated by a clear error so a future-// migration to the Go TOC generator doesn't accidentally fall-// through to chunk iteration.-func TestExtractorComponent_Invoke_TOCNotPorted(t *testing.T) {+// TestExtractorComponent_Invoke_EmptyChunksReturnsEmpty verifies that+// when len(in.chunks) == 0, Invoke immediately returns empty chunks.+func TestExtractorComponent_Invoke_EmptyChunksReturnsEmpty(t *testing.T) {c := &ExtractorComponent{Param: schema.ExtractorParam{- FieldName: "toc",- }}- _, err := c.Invoke(t.Context(), nil, map[string]any{- "chunks": []map[string]any{{"text": "x"}}},- )- if err == nil {- t.Fatal("expected error for field_name=toc, got nil")- }- if !strings.Contains(err.Error(), "toc") {- t.Errorf("error should mention toc: %v", err)- }- if !strings.Contains(err.Error(), "not yet ported") {- t.Errorf("error should call out parity gap: %v", err)- }-}--// TestExtractorComponent_Invoke_NoChunksFastPath verifies the-// no-chunks input still produces a one-element chunks slice-// (mirrors python _invoke line 110 fallback).-func TestExtractorComponent_Invoke_NoChunksFastPath(t *testing.T) {- withStubChatInvoker(t,- stubResponse{Content: "single-shot answer"},- )-- c := &ExtractorComponent{Param: schema.ExtractorParam{- FieldName: "answer",+ Summary: schema.SummaryExtractConfig{Enabled: true},}}out, err := c.Invoke(t.Context(), nil, map[string]any{})if err != nil {@@ -421,21 +273,20 @@ func TestExtractorComponent_Invoke_NoChunksFastPath(t *testing.T) {if !ok {t.Fatalf("chunks missing or wrong shape")}- if len(chunks) != 1 {- t.Fatalf("chunks len = %d, want 1", len(chunks))- }- if chunks[0]["answer"] != "single-shot answer" {- t.Errorf("answer = %v, want %q", chunks[0]["answer"], "single-shot answer")+ if len(chunks) != 0 {+ t.Fatalf("chunks len = %d, want 0", len(chunks))}}+// TestExtractorComponent_Invoke_JSONListInput verifies that the chunks list+// can be provided under the "json" key.func TestExtractorComponent_Invoke_JSONListInput(t *testing.T) {withStubChatInvoker(t,- stubResponse{Content: "json chunk answer"},+ stubResponse{Content: "json chunk summary"},)c := &ExtractorComponent{Param: schema.ExtractorParam{- FieldName: "answer",+ Summary: schema.SummaryExtractConfig{Enabled: true},}}out, err := c.Invoke(t.Context(), nil, map[string]any{"json": []map[string]any{{"text": "json payload chunk"}},@@ -447,25 +298,24 @@ func TestExtractorComponent_Invoke_JSONListInput(t *testing.T) {if !ok || len(chunks) != 1 {t.Fatalf("chunks malformed: %v", out["chunks"])}- if chunks[0]["answer"] != "json chunk answer" {- t.Errorf("answer = %v, want %q", chunks[0]["answer"], "json chunk answer")+ if chunks[0]["summary"] != "json chunk summary" {+ t.Errorf("summary = %v, want %q", chunks[0]["summary"], "json chunk summary")}}// TestExtractorComponent_Invoke_PerCallLLMIDOverride verifies an-// inputs["llm_id"] override wins over Param.LLMID and reaches-// the chat invoker verbatim (the per-call override is the-// explicit test seam for runtime reconfiguration).+// inputs["llm_id"] override wins over Param.LLMID.func TestExtractorComponent_Invoke_PerCallLLMIDOverride(t *testing.T) {stub := withStubChatInvoker(t,- stubResponse{Content: "ok"},+ stubResponse{Content: "summary result"},)c := &ExtractorComponent{Param: schema.ExtractorParam{- FieldName: "out",- LLMID: "static-llm",+ LLMID: "static-llm",+ Summary: schema.SummaryExtractConfig{Enabled: true},}}_, err := c.Invoke(t.Context(), nil, map[string]any{+ "chunks": []map[string]any{{"text": "sample text"}},"llm_id": "override-llm",})if err != nil {@@ -479,19 +329,18 @@ func TestExtractorComponent_Invoke_PerCallLLMIDOverride(t *testing.T) {}// TestExtractorComponent_Invoke_CompositeLLMID verifies the-// composite "gpt-4o-mini@openai" form is split into driver and-// model before reaching the chat invoker. Matches the canonical-// composite llm_id convention used throughout the codebase-// (see internal/agent/component/llm_credentials.go:parseLLMIDParts).+// composite "gpt-4o-mini@openai" form is split into driver and model.func TestExtractorComponent_Invoke_CompositeLLMID(t *testing.T) {stub := withStubChatInvoker(t,- stubResponse{Content: "ok"},+ stubResponse{Content: "summary result"},)c := &ExtractorComponent{Param: schema.ExtractorParam{- FieldName: "out",- LLMID: "gpt-4o-mini@openai",+ LLMID: "gpt-4o-mini@openai",+ Summary: schema.SummaryExtractConfig{Enabled: true},}}- if _, err := c.Invoke(t.Context(), nil, map[string]any{}); err != nil {+ if _, err := c.Invoke(t.Context(), nil, map[string]any{+ "chunks": []map[string]any{{"text": "sample text"}},+ }); err != nil {t.Fatalf("Invoke: %v", err)}stub.mu.Lock()@@ -505,9 +354,7 @@ func TestExtractorComponent_Invoke_CompositeLLMID(t *testing.T) {}// TestExtractorComponent_Invoke_ChunkIndexInError verifies the-// error message includes the failing chunk index so a long-// pipeline run surfaces which input document triggered the LLM-// failure (mirrors python's per-chunk progress call at line 105).+// error message includes the failing chunk index.func TestExtractorComponent_Invoke_ChunkIndexInError(t *testing.T) {prevMax, prevDelay := extractorRetryMax, extractorRetryDelayextractorRetryMax, extractorRetryDelay = 3, time.Millisecond@@ -518,13 +365,13 @@ func TestExtractorComponent_Invoke_ChunkIndexInError(t *testing.T) {errBoom := errors.New("chunk-1-boom")withStubChatInvoker(t,stubResponse{Content: "ok for chunk 0"},- stubResponse{Err: errBoom}, // chunk 1: attempt 0- stubResponse{Err: errBoom}, // attempt 1- stubResponse{Err: errBoom}, // attempt 2- stubResponse{Err: errBoom}, // attempt 3 (last retry)+ stubResponse{Err: errBoom},+ stubResponse{Err: errBoom},+ stubResponse{Err: errBoom},+ stubResponse{Err: errBoom},)c := &ExtractorComponent{Param: schema.ExtractorParam{- FieldName: "out",+ Summary: schema.SummaryExtractConfig{Enabled: true},}}_, err := c.Invoke(t.Context(), nil, map[string]any{"chunks": []map[string]any{@@ -535,17 +382,14 @@ func TestExtractorComponent_Invoke_ChunkIndexInError(t *testing.T) {if err == nil {t.Fatal("Invoke returned nil error")}- if !strings.Contains(err.Error(), "chunk 1") {- t.Errorf("error should mention chunk 1 (zero-indexed): %v", err)+ if !strings.Contains(err.Error(), "chunk 0") && !strings.Contains(err.Error(), "chunk 1") {+ t.Errorf("error should mention failing chunk index: %v", err)}if !strings.Contains(err.Error(), "chunk-1-boom") {t.Errorf("error should chain underlying error: %v", err)}}-// TestExtractorComponent_NewExtractorComponent_ParamCheck covers-// the construction-time Validate() rejection of an empty-// field_name (matches python check_empty "Result Destination").func TestExtractorComponent_NewExtractorComponent_ParamCheck(t *testing.T) {c, err := NewExtractorComponent(map[string]any{})if err != nil {@@ -556,189 +400,23 @@ func TestExtractorComponent_NewExtractorComponent_ParamCheck(t *testing.T) {}}-// TestExtractorComponent_NewExtractorComponent_Happy covers the-// parse path of every supported key; the param block coming out-// should round-trip cleanly through Invoke.func TestExtractorComponent_NewExtractorComponent_Happy(t *testing.T) {- withStubChatInvoker(t, stubResponse{Content: "ok"})c, err := NewExtractorComponent(map[string]any{- "field_name": "summary",- "llm_id": "openai/gpt-4o-mini",- "system_prompt": "You are a precise summarizer.",- "prompt": "Summarize:",- })- if err != nil {- t.Fatalf("NewExtractorComponent: %v", err)- }- if _, err = c.Invoke(t.Context(), nil, map[string]any{- "chunks": []map[string]any{{"text": "x"}}},- ); err != nil {- t.Fatalf("Invoke: %v", err)- }-}--// TestNewExtractorComponent_SysPromptAlias verifies that "sys_prompt"-// (the Python DSL name) is accepted as a fallback for SystemPrompt.-func TestNewExtractorComponent_SysPromptAlias(t *testing.T) {- withStubChatInvoker(t, stubResponse{Content: "ok"})- comp, err := NewExtractorComponent(map[string]any{- "field_name": "out",- "sys_prompt": "You are a Python DSL prompt.",- })- if err != nil {- t.Fatalf("NewExtractorComponent: %v", err)- }- ec := comp.(*ExtractorComponent)- if ec.Param.SystemPrompt != "You are a Python DSL prompt." {- t.Errorf("SystemPrompt = %q, want %q", ec.Param.SystemPrompt, "You are a Python DSL prompt.")- }-}--// TestNewExtractorComponent_MetadataAsAnySlice guards against the regression-// where InjectExtractorEnableMetadata injected the field schema as a-// []map[string]interface{} while NewExtractorComponent only accepted []any;-// the type assertion then failed and ExtractorParam.Metadata stayed empty, so-// auto-metadata never fired. The override_params path passes the injected-// value straight through (no JSON round-trip), so the slice element type must-// be []any for the assertion to succeed.-func TestNewExtractorComponent_MetadataAsAnySlice(t *testing.T) {- comp, err := NewExtractorComponent(map[string]any{- "field_name": "out",- "enable_metadata": 1,- // This is exactly the dynamic type InjectExtractorEnableMetadata- // produces ([]any of map[string]any), NOT []map[string]interface{}.- "metadata": []any{- map[string]any{"key": "author", "type": "string", "description": "doc author"},- map[string]any{"key": "year", "type": "number", "enum": []any{"2020", "2021"}},- },- })- if err != nil {- t.Fatalf("NewExtractorComponent: %v", err)- }- ec := comp.(*ExtractorComponent)- if ec.Param.EnableMetadata != 1 {- t.Fatalf("EnableMetadata = %d, want 1", ec.Param.EnableMetadata)- }- if len(ec.Param.Metadata) != 2 {- t.Fatalf("Metadata = %#v, want 2 fields", ec.Param.Metadata)- }- if ec.Param.Metadata[0].Key != "author" || ec.Param.Metadata[0].Type != "string" {- t.Errorf("Metadata[0] = %#v, want key=author type=string", ec.Param.Metadata[0])- }- if len(ec.Param.Metadata[1].Enum) != 2 {- t.Errorf("Metadata[1].Enum = %#v, want 2 enum values", ec.Param.Metadata[1].Enum)- }-}--// TestNewExtractorComponent_PromptsArray verifies that the Python DSL-// "prompts" array format is parsed into Param.Prompt.-func TestNewExtractorComponent_PromptsArray(t *testing.T) {- withStubChatInvoker(t, stubResponse{Content: "ok"})- comp, err := NewExtractorComponent(map[string]any{- "field_name": "out",- "prompts": []any{- map[string]any{- "content": "Analyze: {TitleChunker:FlatMiceFix@chunks}",- "role": "user",- },- },- })- if err != nil {- t.Fatalf("NewExtractorComponent: %v", err)- }- ec := comp.(*ExtractorComponent)- want := "Analyze: {TitleChunker:FlatMiceFix@chunks}"- if ec.Param.Prompt != want {- t.Errorf("Prompt = %q, want %q", ec.Param.Prompt, want)- }-}--// TestNewExtractorComponent_PromptsArray_PromptWins verifies that-// "prompt" (string) takes priority over "prompts" (array) when both-// are present in the DSL params.-func TestNewExtractorComponent_PromptsArray_PromptWins(t *testing.T) {- withStubChatInvoker(t, stubResponse{Content: "ok"})- comp, err := NewExtractorComponent(map[string]any{- "field_name": "out",- "prompt": "Direct prompt wins.",- "prompts": []any{- map[string]any{- "content": "Should be ignored.",- "role": "user",- },+ "llm_id": "openai/gpt-4o-mini",+ "summary": map[string]any{+ "enabled": true,},})if err != nil {t.Fatalf("NewExtractorComponent: %v", err)}- ec := comp.(*ExtractorComponent)- if ec.Param.Prompt != "Direct prompt wins." {- t.Errorf("Prompt = %q, want %q", ec.Param.Prompt, "Direct prompt wins.")- }-}--// TestNewExtractorComponent_PromptsString verifies that a bare-string-// "prompts" (the shape emitted by the front-end graph.nodes form and-// the dsl/testdata templates) is normalized into Param.Prompt, mirroring-// Python agent/component/llm.py:119-120 which coerces a string prompts-// into [{"role":"user","content":prompts}]. Without this normalization-// the string form is silently dropped (the .([]any) assertion fails).-func TestNewExtractorComponent_PromptsString(t *testing.T) {- withStubChatInvoker(t, stubResponse{Content: "ok"})- comp, err := NewExtractorComponent(map[string]any{- "field_name": "out",- "prompts": "Content: {TitleChunker:FlatMiceFix@chunks}",- })- if err != nil {- t.Fatalf("NewExtractorComponent: %v", err)- }- ec := comp.(*ExtractorComponent)- want := "Content: {TitleChunker:FlatMiceFix@chunks}"- if ec.Param.Prompt != want {- t.Errorf("Prompt = %q, want %q (string prompts should be normalized)", ec.Param.Prompt, want)- }-}--// TestNewExtractorComponent_PromptsString_PromptWins verifies that-// "prompt" (string) still takes priority over a string-form "prompts"-// when both are present, matching the prompt>prompts precedence of-// the list-form path (TestNewExtractorComponent_PromptsArray_PromptWins).-func TestNewExtractorComponent_PromptsString_PromptWins(t *testing.T) {- withStubChatInvoker(t, stubResponse{Content: "ok"})- comp, err := NewExtractorComponent(map[string]any{- "field_name": "out",- "prompt": "Direct prompt wins.",- "prompts": "Should be ignored.",- })- if err != nil {- t.Fatalf("NewExtractorComponent: %v", err)- }- ec := comp.(*ExtractorComponent)- if ec.Param.Prompt != "Direct prompt wins." {- t.Errorf("Prompt = %q, want %q", ec.Param.Prompt, "Direct prompt wins.")- }-}--// TestNewExtractorComponent_SystemPromptWinsOverSysPrompt verifies-// that "system_prompt" takes priority over "sys_prompt".-func TestNewExtractorComponent_SystemPromptWinsOverSysPrompt(t *testing.T) {- withStubChatInvoker(t, stubResponse{Content: "ok"})- comp, err := NewExtractorComponent(map[string]any{- "field_name": "out",- "system_prompt": "system_prompt wins.",- "sys_prompt": "sys_prompt ignored.",- })- if err != nil {- t.Fatalf("NewExtractorComponent: %v", err)- }- ec := comp.(*ExtractorComponent)- if ec.Param.SystemPrompt != "system_prompt wins." {- t.Errorf("SystemPrompt = %q, want %q", ec.Param.SystemPrompt, "system_prompt wins.")+ ext := c.(*ExtractorComponent)+ if !ext.Param.Summary.Enabled || ext.Param.LLMID != "openai/gpt-4o-mini" {+ t.Errorf("unexpected params: %+v", ext.Param)}}-// TestExtractorComponent_InputsOutputs_NonEmpty is the shape-// assertion Phase 4's API endpoint relies on.+// TestExtractorComponent_InputsOutputs_NonEmpty verifies Inputs and Outputs shapes.func TestExtractorComponent_InputsOutputs_NonEmpty(t *testing.T) {c := &ExtractorComponent{}ins := c.Inputs()@@ -746,6 +424,18 @@ func TestExtractorComponent_InputsOutputs_NonEmpty(t *testing.T) {if len(ins) == 0 {t.Error("Inputs() returned empty map")}+ if _, ok := ins["chunks"]; !ok {+ t.Errorf("Inputs() missing %q", "chunks")+ }+ if _, ok := ins["llm_id"]; !ok {+ t.Errorf("Inputs() missing %q", "llm_id")+ }+ if _, ok := ins["prompt"]; ok {+ t.Errorf("Inputs() should not contain deprecated %q", "prompt")+ }+ if _, ok := ins["system_prompt"]; ok {+ t.Errorf("Inputs() should not contain deprecated %q", "system_prompt")+ }if len(outs) == 0 {t.Error("Outputs() returned empty map")}@@ -757,11 +447,7 @@ func TestExtractorComponent_InputsOutputs_NonEmpty(t *testing.T) {}}-// TestSplitExtractorLLID covers the composite-id parser in-// isolation — keeps the matrix of edge cases at one call site-// so a regression is easy to attribute. The "@" separator is-// the canonical composite llm_id form used throughout the-// codebase (see internal/agent/component/llm_credentials.go).+// TestSplitExtractorLLID covers the composite-id parser in isolation.func TestSplitExtractorLLID(t *testing.T) {cases := []struct {in string@@ -791,25 +477,19 @@ func TestSplitExtractorLLID(t *testing.T) {}}-// TestTryParseJSONObject covers the best-effort JSON parser-// independently of the LLM seam so its matrix of edge cases is-// easy to attribute.+// TestTryParseJSONObject covers the JSON parser.func TestTryParseJSONObject(t *testing.T) {cases := []struct {name stringin stringwantOK bool- wantKey string // when wantOK=true, expected key in the parsed map+ wantKey string}{{name: "object", in: `{"a":1}`, wantOK: true, wantKey: "a"},{name: "object with fence", in: "```json\n{\"a\":1}\n```", wantOK: true, wantKey: "a"},{name: "fence without json tag", in: "```\n{\"a\":1}\n```", wantOK: true, wantKey: "a"},- // Language tag on its own line (```\njson\n{...}) — Python json_repair- // tolerates this, so encoding/json must not choke on the bare "json".{name: "json tag on own line", in: "```\njson\n{\"a\":1}\n```", wantOK: true, wantKey: "a"},{name: "JSON tag on own line", in: "```\nJSON\n{\"a\":1}\n```", wantOK: true, wantKey: "a"},- // Leading prose before the fence must not be stripped (only a real- // ``` fence prefix is handled).{name: "leading prose no fence", in: "Here is the result: {\"a\":1}", wantOK: false},{name: "plain string", in: "hello", wantOK: false},{name: "array", in: `[1,2]`, wantOK: false},@@ -831,8 +511,7 @@ func TestTryParseJSONObject(t *testing.T) {}}-// TestCleanExtractionResult covers the </think> chain-of-thought stripping-// and the **ERROR** guard that mirrors Python's metadata post-processing.+// TestCleanExtractionResult covers think tag and error marker stripping.func TestCleanExtractionResult(t *testing.T) {cases := []struct {name string@@ -840,11 +519,8 @@ func TestCleanExtractionResult(t *testing.T) {want string}{{name: "plain", in: `{"a":1}`, want: `{"a":1}`},- // Python re.sub(r"^.*</think>", "", ans): everything up to and- // including the LAST </think> is dropped.{name: "thinks stripped", in: "let me think<think>reasoning</think>\n{\"a\":1}", want: `{"a":1}`},{name: "thinks no json", in: "thinking</think>no json here", want: "no json here"},- // **ERROR** responses are rejected entirely.{name: "error marker rejected", in: "**ERROR** could not extract", want: ""},{name: "error after think", in: "x</think>**ERROR** boom", want: ""},{name: "whitespace trimmed", in: " {\"a\":1} ", want: `{"a":1}`},@@ -862,14 +538,15 @@ func TestCleanExtractionResult(t *testing.T) {// metadata extraction with the given field definitions.func newMetadataExtractor(fields ...common.MetadataFieldDef) *ExtractorComponent {return &ExtractorComponent{Param: schema.ExtractorParam{- EnableMetadata: 1,- Metadata: fields,+ Metadata: schema.MetadataExtractConfig{+ Enabled: true,+ Metadata: fields,+ },}}}// TestExtractorComponent_runEnableMetadata_MergesIntoChunkMetadata verifies a-// JSON object from the LLM is parsed and merged into the chunk's metadata map,-// which mergeChunkMetadata then aggregates to the doc level.+// JSON object from the LLM is parsed and merged into the chunk's metadata map.func TestExtractorComponent_runEnableMetadata_MergesIntoChunkMetadata(t *testing.T) {withStubChatInvoker(t, stubResponse{Content: `{"category":"finance","region":"east"}`})c := newMetadataExtractor(@@ -890,9 +567,7 @@ func TestExtractorComponent_runEnableMetadata_MergesIntoChunkMetadata(t *testing}// TestExtractorComponent_runEnableMetadata_StripsJSONFence verifies the-// extraction path tolerates a fenced ```json response (the common model-// output) that would otherwise fail encoding/json parsing — mirroring Python-// json_repair.+// extraction path tolerates a fenced ```json response.func TestExtractorComponent_runEnableMetadata_StripsJSONFence(t *testing.T) {withStubChatInvoker(t, stubResponse{Content: "```json\n{\"category\":\"law\"}\n```"})c := newMetadataExtractor(common.MetadataFieldDef{Key: "category", Type: "string"})@@ -910,10 +585,7 @@ func TestExtractorComponent_runEnableMetadata_StripsJSONFence(t *testing.T) {}// TestExtractorComponent_runEnableMetadata_MidTextThink verifies the full-// metadata path — LLM call, second-layer <think> strip, JSON parse, merge —-// tolerates a mid-text reasoning block preceded by a preamble, matching-// Python gen_metadata. This is the end-to-end guard for callStructured's-// common.StripThinkTrailing: without it the metadata extraction silently drops.+// metadata path tolerates a mid-text reasoning block preceded by a preamble.func TestExtractorComponent_runEnableMetadata_MidTextThink(t *testing.T) {withStubChatInvoker(t, stubResponse{Content: `preamble<think>reasoning</think>{"category":"finance"}`})c := newMetadataExtractor(common.MetadataFieldDef{Key: "category", Type: "string"})@@ -931,9 +603,7 @@ func TestExtractorComponent_runEnableMetadata_MidTextThink(t *testing.T) {}// TestExtractorComponent_runEnableMetadata_DegradesGracefully verifies that an-// empty / **ERROR** / unparseable / think-only LLM response does NOT block-// ingestion: the chunk metadata is left untouched and no error is returned-// (Python "no evidence → {}").+// empty / **ERROR** / unparseable / think-only LLM response does NOT block ingestion.func TestExtractorComponent_runEnableMetadata_DegradesGracefully(t *testing.T) {cases := []struct {name string@@ -968,9 +638,7 @@ func TestExtractorComponent_runEnableMetadata_DegradesGracefully(t *testing.T) {}// TestExtractorComponent_runEnableMetadata_CrossChunkUnion simulates two chunks-// whose extraction returns overlapping list values for the same key. Aggregating-// the chunk metadata maps with utility.UpdateMetadataTo (as mergeChunkMetadata-// does) must produce a de-duplicated union, matching Python update_metadata_to.+// whose extraction returns overlapping list values for the same key.func TestExtractorComponent_runEnableMetadata_CrossChunkUnion(t *testing.T) {withStubChatInvoker(t,stubResponse{Content: `{"people":["关羽","张辽"]}`},@@ -985,7 +653,6 @@ func TestExtractorComponent_runEnableMetadata_CrossChunkUnion(t *testing.T) {if err := c.runEnableMetadata(t.Context(), nil, extractorInputs{llmID: "m"}, ck2, "chunk two"); err != nil {t.Fatalf("ck2: %v", err)}- // mirror mergeChunkMetadata: aggregate chunk metadata into doc metadata.m1, ok := ck1["metadata"].(map[string]any)if !ok {t.Fatalf("ck1[metadata] missing: %T", ck1["metadata"])@@ -1014,8 +681,7 @@ func TestExtractorComponent_runEnableMetadata_CrossChunkUnion(t *testing.T) {// TestExtractorComponent_runEnableMetadata_CombinedValueSplit verifies a value// the LLM combines with Chinese/comma delimiters is split when passed through-// common.SplitCombinedMetadataValues (as mergeDocMetadata does before writing),-// matching Python _split_combined_values (doc_metadata_service.py).+// common.SplitCombinedMetadataValues.func TestExtractorComponent_runEnableMetadata_CombinedValueSplit(t *testing.T) {withStubChatInvoker(t, stubResponse{Content: `{"people":["关羽、张辽、刘备"]}`})c := newMetadataExtractor(common.MetadataFieldDef{Key: "people", Type: "string"})@@ -1031,7 +697,6 @@ func TestExtractorComponent_runEnableMetadata_CombinedValueSplit(t *testing.T) {if !ok || len(raw) != 1 {t.Fatalf("raw people = %v, want 1 combined element", rawMeta["people"])}- // mergeDocMetadata runs SplitCombinedMetadataValues before writing.split := common.SplitCombinedMetadataValues(ck["metadata"].(map[string]any))people, ok := split["people"].([]string)if !ok {@@ -1049,9 +714,7 @@ func TestExtractorComponent_runEnableMetadata_CombinedValueSplit(t *testing.T) {}// TestExtractorComponent_ConcurrentInvoke verifies the chat-// invoker swap is safe under concurrent Invoke calls. This is-// the canary for SetExtractorChatInvoker and the package-level-// RWMutex contract — a data race here breaks race detector.+// invoker swap is safe under concurrent Invoke calls.func TestExtractorComponent_ConcurrentInvoke(t *testing.T) {withStubChatInvoker(t,stubResponse{Content: "1"},@@ -1060,7 +723,7 @@ func TestExtractorComponent_ConcurrentInvoke(t *testing.T) {stubResponse{Content: "4"},)c := &ExtractorComponent{Param: schema.ExtractorParam{- FieldName: "out",+ Summary: schema.SummaryExtractConfig{Enabled: true},}}chunks := []map[string]any{{"text": "a"}, {"text": "b"}, {"text": "c"}, {"text": "d"},@@ -1086,10 +749,6 @@ func TestExtractorComponent_ConcurrentInvoke(t *testing.T) {}}-// silence unused-import vet warnings for eschema in case the-// test file is built without the import ever being referenced-// (it currently isn't, but pinning the import keeps test-side-// imports honest if helpers move around in future revisions).var _ = eschema.Message{}// TestIsBareTenantModelID verifies UUID detection.@@ -1100,8 +759,8 @@ func TestIsBareTenantModelID(t *testing.T) {}{{"9e819c2442b14f9dab46062916e29195", true},{"ABCDEFabcdef01234567890123456789", true},- {"9e819c2442b14f9dab46062916e2919", false}, // 31 chars- {"9e819c2442b14f9dab46062916e29195X", false}, // 33 chars+ {"9e819c2442b14f9dab46062916e2919", false},+ {"9e819c2442b14f9dab46062916e29195X", false},{"gpt-4o-mini@openai", false},{"", false},{"not-a-uuid", false},@@ -1114,8 +773,7 @@ func TestIsBareTenantModelID(t *testing.T) {}}-// TestResolveExtractorChatTarget_AtSplitFallback verifies the @ split-// fallback path works without canvas state (unit test compatibility).+// TestResolveExtractorChatTarget_AtSplitFallback verifies the @ split fallback.func TestResolveExtractorChatTarget_AtSplitFallback(t *testing.T) {ctx := t.Context()driver, modelName, apiKey, baseURL, err := resolveExtractorChatTarget(@@ -1134,8 +792,7 @@ func TestResolveExtractorChatTarget_AtSplitFallback(t *testing.T) {}}-// TestResolveExtractorChatTarget_NoDriver verifies a non-@ plain string-// without canvas state returns no driver (passes through to Chat()).+// TestResolveExtractorChatTarget_NoDriver verifies a non-@ plain string returns no driver.func TestResolveExtractorChatTarget_NoDriver(t *testing.T) {ctx := t.Context()driver, modelName, _, _, err := resolveExtractorChatTarget(@@ -1151,20 +808,15 @@ func TestResolveExtractorChatTarget_NoDriver(t *testing.T) {}}-// TestExtractorComponent_Invoke_TemperatureSet verifies the keyword-// extraction LLM chat call receives Temperature=0.2, matching Python's-// keyword_extraction and question_proposal defaults (generator.py:230,245).-// Field extraction intentionally runs on a separate call and uses the-// model default (see TestExtractorComponent_Invoke_FieldNameTemperatureDefault),-// so this test enables only AutoKeywords to assert the 0.2 pin directly.+// TestExtractorComponent_Invoke_TemperatureSet verifies keyword extraction receives Temperature=0.2.func TestExtractorComponent_Invoke_TemperatureSet(t *testing.T) {stub := withStubChatInvoker(t,stubResponse{Content: "keyword, extraction"},)c := &ExtractorComponent{Param: schema.ExtractorParam{- LLMID: "gpt-4o-mini",- AutoKeywords: 3,+ LLMID: "gpt-4o-mini",+ Keywords: schema.KeywordExtractConfig{TopN: 3},}}_, err := c.Invoke(t.Context(), nil, map[string]any{"chunks": []map[string]any{{"text": "document content"}},@@ -1186,37 +838,7 @@ func TestExtractorComponent_Invoke_TemperatureSet(t *testing.T) {}}-// TestExtractorComponent_Invoke_FieldNameTemperatureDefault verifies-// that the generic field-extraction path leaves Temperature unset-// (model/default), unlike keyword/question which pin 0.2 — matching-// Python's generic Extractor behavior.-func TestExtractorComponent_Invoke_FieldNameTemperatureDefault(t *testing.T) {- stub := withStubChatInvoker(t,- stubResponse{Content: "extracted"},- )-- c := &ExtractorComponent{Param: schema.ExtractorParam{- FieldName: "summary",- LLMID: "gpt-4o-mini",- }}- _, err := c.Invoke(t.Context(), nil, map[string]any{- "chunks": []map[string]any{{"text": "document content"}},- })- if err != nil {- t.Fatalf("Invoke: %v", err)- }-- stub.mu.Lock()- defer stub.mu.Unlock()- if stub.lastRequest().Temperature != nil {- t.Errorf("Temperature = %v, want nil (field extraction uses model default)", *stub.lastRequest().Temperature)- }-}--// TestIsRetryableLLMError locks in the retry-classification heuristic,-// especially the word-boundary guard that prevents a transient timeout-// message ("...after 400ms") from being misclassified as a permanent-// HTTP 400 and dropped.+// TestIsRetryableLLMError tests the retry classification heuristic.func TestIsRetryableLLMError(t *testing.T) {tests := []struct {name string@@ -1251,41 +873,18 @@ func TestIsRetryableLLMError(t *testing.T) {}}-// TestCleanExtractionResult_LastThinkTag verifies that when the LLM-// response contains multiple </think> tags, cleanExtractionResult strips-// up to the LAST one (greedy, matching Python's re.sub), not just the-// first (which would leave a residual think block in the output).+// TestCleanExtractionResult_LastThinkTag verifies think tag removal.func TestCleanExtractionResult_LastThinkTag(t *testing.T) {tests := []struct {name stringin stringwant string}{- {- name: "single think block",- in: "<think>reasoning</think>the answer",- want: "the answer",- },- {- name: "nested think blocks",- in: "<think>outer</think>mid<think>inner</think>final output",- want: "final output",- },- {- name: "no think tag",- in: "plain answer",- want: "plain answer",- },- {- name: "think tag without close",- in: "<think>unclosed",- want: "<think>unclosed",- },- {- name: "error sentinel",- in: "valid output**ERROR**extra",- want: "",- },+ {name: "single think block", in: "<think>reasoning</think>the answer", want: "the answer"},+ {name: "nested think blocks", in: "<think>outer</think>mid<think>inner</think>final output", want: "final output"},+ {name: "no think tag", in: "plain answer", want: "plain answer"},+ {name: "think tag without close", in: "<think>unclosed", want: "<think>unclosed"},+ {name: "error sentinel", in: "valid output**ERROR**extra", want: ""},}for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {@@ -1297,10 +896,7 @@ func TestCleanExtractionResult_LastThinkTag(t *testing.T) {}}-// TestCleanLLMText verifies the two-step LLM-layer cleanup that Python-// applies in LLMBundle.async_chat (llm_service.py:459-461): reasoning-// content is stripped only when a leading <think> has a matching closing-// </think> after it, and <tool_call>...</tool_call> blocks are removed.+// TestCleanLLMText verifies cleanLLMText reasoning tag and tool call removal.func TestCleanLLMText(t *testing.T) {tests := []struct {name string@@ -1315,17 +911,17 @@ func TestCleanLLMText(t *testing.T) {{name: "close without open kept",in: "abc</think>def",- want: "abc</think>def", // no leading <think> → unchanged+ want: "abc</think>def",},{name: "prefix before think kept",in: "prefix<think>reason</think>answer",- want: "prefix<think>reason</think>answer", // <think> not at start → content preserved+ want: "prefix<think>reason</think>answer",},{name: "open without close kept",in: "<think>unclosed",- want: "<think>unclosed", // no </think> → unchanged+ want: "<think>unclosed",},{name: "tool_call block removed",@@ -1357,13 +953,11 @@ func TestCleanLLMText(t *testing.T) {}}-// TestExtractorComponent_callStructured verifies the metadata path parses a-// JSON object response into a map, and returns (nil, nil) for a non-JSON or-// empty response (nothing extracted, not an error).+// TestExtractorComponent_callStructured verifies structured parsing.func TestExtractorComponent_callStructured(t *testing.T) {withStubChatInvoker(t, stubResponse{Content: `{"a": 1}`})c := &ExtractorComponent{}- got, err := c.callStructured(t.Context(), nil, extractorInputs{llmID: "m"}, "")+ got, err := c.callStructured(t.Context(), nil, extractorInputs{llmID: "m"}, "system", "")if err != nil {t.Fatalf("callStructured: %v", err)}@@ -1373,7 +967,7 @@ func TestExtractorComponent_callStructured(t *testing.T) {// Non-JSON response → (nil, nil), not an error.withStubChatInvoker(t, stubResponse{Content: "this is not JSON"})- got, err = c.callStructured(t.Context(), nil, extractorInputs{llmID: "m"}, "")+ got, err = c.callStructured(t.Context(), nil, extractorInputs{llmID: "m"}, "system", "")if err != nil {t.Fatalf("callStructured on non-JSON: %v", err)}@@ -1382,16 +976,11 @@ func TestExtractorComponent_callStructured(t *testing.T) {}}-// TestExtractorComponent_callStructured_MidTextThink verifies the metadata-// path's second cleanup layer (common.StripThinkTrailing) strips a mid-text-// reasoning block preceded by a preamble, matching Python's gen_metadata-// double cleanup (async_chat + re.sub r"^.*</think>"). Without it the JSON-// would survive the leading-only cleanLLMText, fail to parse, and silently-// drop the metadata extraction.+// TestExtractorComponent_callStructured_MidTextThink verifies think trailing stripping.func TestExtractorComponent_callStructured_MidTextThink(t *testing.T) {withStubChatInvoker(t, stubResponse{Content: `preamble<think>reasoning</think>{"a": 1}`})c := &ExtractorComponent{}- got, err := c.callStructured(t.Context(), nil, extractorInputs{llmID: "m"}, "")+ got, err := c.callStructured(t.Context(), nil, extractorInputs{llmID: "m"}, "system", "")if err != nil {t.Fatalf("callStructured: %v", err)}@@ -1401,9 +990,7 @@ func TestExtractorComponent_callStructured_MidTextThink(t *testing.T) {}// TestExtractorComponent_Invoke_ConcurrentKeywordsAndQuestions verifies-// that when both auto_keywords and auto_questions are enabled, both-// LLM calls are dispatched per chunk and results land on the chunk-// (matching Python's ThreadPoolExecutor concurrency: task_executor.py:444-448).+// keyword and question extraction on chunks.func TestExtractorComponent_Invoke_ConcurrentKeywordsAndQuestions(t *testing.T) {stub := withStubChatInvoker(t,stubResponse{Content: "alpha, beta"}, // chunk 0 keywords@@ -1413,9 +1000,9 @@ func TestExtractorComponent_Invoke_ConcurrentKeywordsAndQuestions(t *testing.T))c := &ExtractorComponent{Param: schema.ExtractorParam{- LLMID: "gpt-4o-mini",- AutoKeywords: 2,- AutoQuestions: 2,+ LLMID: "gpt-4o-mini",+ Keywords: schema.KeywordExtractConfig{TopN: 2},+ Questions: schema.QuestionExtractConfig{TopN: 2},}}out, err := c.Invoke(t.Context(), nil, map[string]any{"chunks": []map[string]any{@@ -1432,7 +1019,6 @@ func TestExtractorComponent_Invoke_ConcurrentKeywordsAndQuestions(t *testing.T)t.Fatalf("expected 2 chunks, got %v", out["chunks"])}- // Both chunks should have keywords and questions populated.for i, ck := range chunks {kwds, hasKW := ck["important_kwd"].([]string)if !hasKW || len(kwds) == 0 {@@ -1449,455 +1035,101 @@ func TestExtractorComponent_Invoke_ConcurrentKeywordsAndQuestions(t *testing.T)}}-// TestResolveExtractorChatTarget_EmptyLLMID verifies that when llmID is-// empty, resolveExtractorChatTarget falls back to the tenant default chat-// model (via resolveTenantModelByType), matching Python's behavior-// (task_executor.py:573-574 never skips tagging on empty llm_id).-// When no canvas state is available (unit-test context), returns empty-// driver — callers like runAutoTags check driver!="" before using it.+// TestResolveExtractorChatTarget_EmptyLLMID verifies default fallback when llmID is empty.func TestResolveExtractorChatTarget_EmptyLLMID(t *testing.T) {- // Without canvas state: empty llmID returns empty driver (no crash).ctx := t.Context()driver, modelName, _, _, err := resolveExtractorChatTarget(ctx, dao.DB, "")if err != nil {t.Fatalf("unexpected error: %v", err)}- // In test context without canvas state, neither tenant default nor @ split- // can resolve — driver ends up empty. Callers must handle this gracefully.if driver != "" {- t.Logf("resolved empty llmID: driver=%q model=%q (tenant default might be available)", driver, modelName)+ t.Logf("resolved empty llmID: driver=%q model=%q", driver, modelName)}- // Contract: no panic, no error for empty llmID.}-// TestExtractorComponent_Invoke_ContentWithWeightPlaceholder verifies that-// a prompt referencing {content_with_weight} (a chunk field that is NOT in-// the {text}/{chunks} suppression set of the old code) substitutes the-// field without also appending the chunk text a second time. Regression-// guard for the duplicate-injection bug fixed in-// fix/extractor-chunk-text-injection.-func TestExtractorComponent_Invoke_ContentWithWeightPlaceholder(t *testing.T) {- stub := withStubChatInvoker(t,- stubResponse{Content: "answer"},- )-- c := &ExtractorComponent{Param: schema.ExtractorParam{- FieldName: "out",- Prompt: "Weighted: {content_with_weight}",- LLMID: "gpt-4o-mini",- }}- _, err := c.Invoke(t.Context(), nil, map[string]any{- "chunks": []map[string]any{{"content_with_weight": "weighted doc"}},- })- if err != nil {- t.Fatalf("Invoke: %v", err)- }+// TestFitExtractorMessages_RejectsEmptyUserTurn verifies rejection of emptied user turn.+func TestFitExtractorMessages_RejectsEmptyUserTurn(t *testing.T) {+ SetExtractorContextLengthOverride(func(_ context.Context, _ string) int { return 500 })+ t.Cleanup(func() { SetExtractorContextLengthOverride(nil) })- stub.mu.Lock()- defer stub.mu.Unlock()- var userContent string- for _, msg := range stub.lastRequest().Messages {- if msg.Role == eschema.User {- userContent = msg.Content- }- }- if strings.Contains(userContent, "{content_with_weight}") {- t.Errorf("prompt still contains literal {content_with_weight}: %q", userContent)+ msgs := []eschema.Message{+ {Role: eschema.System, Content: strings.Repeat("s ", 1000)},+ {Role: eschema.User, Content: strings.Repeat("u ", 400)},}- if n := strings.Count(userContent, "weighted doc"); n != 1 {- t.Errorf("chunk text appears %d times, want 1 (no duplicate append): %q", n, userContent)+ if _, err := fitExtractorMessages(t.Context(), nil, "test@test", msgs); err == nil {+ t.Fatal("expected an error when fitting empties the user turn")}}-// TestExtractorComponent_Invoke_NonContentPlaceholderKeepsChunkText verifies-// that a non-content placeholder like {title} being substituted does NOT-// suppress the chunk-text append — otherwise the document body would-// silently disappear from the LLM call. Regression guard for the-// "compare substituted vs original" approach, which incorrectly suppressed-// on any replacement.-func TestExtractorComponent_Invoke_NonContentPlaceholderKeepsChunkText(t *testing.T) {- stub := withStubChatInvoker(t,- stubResponse{Content: "answer"},- )+// TestFitExtractorMessages_KeepsUserTurn verifies prompt trimming happy path.+func TestFitExtractorMessages_KeepsUserTurn(t *testing.T) {+ SetExtractorContextLengthOverride(func(_ context.Context, _ string) int { return 2000 })+ t.Cleanup(func() { SetExtractorContextLengthOverride(nil) })- c := &ExtractorComponent{Param: schema.ExtractorParam{- FieldName: "out",- Prompt: "Title: {title}\nExtract:",- LLMID: "gpt-4o-mini",- }}- _, err := c.Invoke(t.Context(), nil, map[string]any{- "chunks": []map[string]any{{- "text": "DOC BODY",- "title": "My Title",- }},- })- if err != nil {- t.Fatalf("Invoke: %v", err)+ msgs := []eschema.Message{+ {Role: eschema.System, Content: "you are a helpful assistant"},+ {Role: eschema.User, Content: strings.Repeat("u ", 3000)},}-- stub.mu.Lock()- defer stub.mu.Unlock()- var userContent string- for _, msg := range stub.lastRequest().Messages {- if msg.Role == eschema.User {- userContent = msg.Content- }+ fitted, err := fitExtractorMessages(t.Context(), nil, "test@test", msgs)+ if err != nil {+ t.Fatalf("unexpected error: %v", err)}- // {title} must be replaced, and chunk body must still be present.- if strings.Contains(userContent, "{title}") {- t.Errorf("prompt still contains literal {title}: %q", userContent)+ if len(fitted) != 2 {+ t.Fatalf("got %d messages, want 2", len(fitted))}- if !strings.Contains(userContent, "DOC BODY") {- t.Errorf("chunk body missing from LLM call — append was wrongly suppressed: %q", userContent)+ if strings.TrimSpace(fitted[1].Content) == "" {+ t.Fatal("user turn was emptied")}}-// TestExtractorComponent_Invoke_UnresolvedTextPlaceholderKeepsChunkText verifies-// that a {text} placeholder that cannot be resolved against the chunk (the-// chunk has content_with_weight but no text field) does NOT suppress the-// chunk-text append. Otherwise the LLM receives a literal {text} and no content.-func TestExtractorComponent_Invoke_UnresolvedTextPlaceholderKeepsChunkText(t *testing.T) {- stub := withStubChatInvoker(t,- stubResponse{Content: "answer"},- )+// TestFitExtractorMessages_NoSystemPromptKeepsUserTurn verifies user-only message fitting.+func TestFitExtractorMessages_NoSystemPromptKeepsUserTurn(t *testing.T) {+ SetExtractorContextLengthOverride(func(_ context.Context, _ string) int { return 2000 })+ t.Cleanup(func() { SetExtractorContextLengthOverride(nil) })- c := &ExtractorComponent{Param: schema.ExtractorParam{- FieldName: "out",- Prompt: "Content: {text}",- LLMID: "gpt-4o-mini",- }}- _, err := c.Invoke(t.Context(), nil, map[string]any{- "chunks": []map[string]any{{- "content_with_weight": "weighted doc",- }},- })+ msgs := []eschema.Message{+ {Role: eschema.User, Content: strings.Repeat("u ", 3000)},+ }+ fitted, err := fitExtractorMessages(t.Context(), nil, "test@test", msgs)if err != nil {- t.Fatalf("Invoke: %v", err)+ t.Fatalf("unexpected error for user-only prompt: %v", err)}-- stub.mu.Lock()- defer stub.mu.Unlock()- var userContent string- for _, msg := range stub.lastRequest().Messages {- if msg.Role == eschema.User {- userContent = msg.Content- }+ if len(fitted) != 1 || fitted[0].Role != eschema.User {+ t.Fatalf("got %d messages, want the single user turn: %+v", len(fitted), fitted)}- // {text} was not resolved (chunk has no text field), so the append must- // still deliver the chunk body.- if !strings.Contains(userContent, "weighted doc") {- t.Errorf("chunk body missing — append was wrongly suppressed on unresolved {text}: %q", userContent)+ if strings.TrimSpace(fitted[0].Content) == "" {+ t.Fatal("user turn was emptied")}}-// TestExtractorComponent_Invoke_SubstitutesPlaceholders verifies that-// {field_name} placeholders in the user prompt are substituted with-// the current chunk's field values before the LLM call, matching-// Python's string_format (agent/component/base.py:602).-func TestExtractorComponent_Invoke_SubstitutesPlaceholders(t *testing.T) {- stub := withStubChatInvoker(t,- stubResponse{Content: "substituted answer"},- )+// TestExtractorComponent_CallRaw_FitsBeforeInvoke verifies message fitting end to end.+func TestExtractorComponent_CallRaw_FitsBeforeInvoke(t *testing.T) {+ SetExtractorContextLengthOverride(func(_ context.Context, _ string) int { return 200 })+ t.Cleanup(func() { SetExtractorContextLengthOverride(nil) })- c := &ExtractorComponent{Param: schema.ExtractorParam{- FieldName: "summary",- Prompt: "Analyze: {text}",- LLMID: "gpt-4o-mini",- }}- _, err := c.Invoke(t.Context(), nil, map[string]any{- "chunks": []map[string]any{{"text": "the document content"}},- })+ stub := withStubChatInvoker(t, stubResponse{Content: `{"ok": true}`})+ c := &ExtractorComponent{}++ chunkBody := strings.Repeat("chunk text with lots of tokens. ", 500)+ _, err := c.callText(t.Context(), nil, extractorInputs{+ llmID: "test@test",+ }, "extract fields", chunkBody)if err != nil {- t.Fatalf("Invoke: %v", err)+ t.Fatalf("callText: %v", err)}stub.mu.Lock()- defer stub.mu.Unlock()- var userContent string- for _, msg := range stub.lastRequest().Messages {- if msg.Role == eschema.User {- userContent = msg.Content- }+ req := stub.lastRequest()+ stub.mu.Unlock()+ if len(req.Messages) == 0 {+ t.Fatal("invoker was not called")}- if strings.Contains(userContent, "{text}") {- t.Errorf("prompt still contains literal {text}: %q", userContent)+ if req.Messages[0].Role != eschema.System || strings.TrimSpace(req.Messages[0].Content) == "" {+ t.Fatalf("system prompt lost or emptied before invoke: %+v", req.Messages[0])}- if !strings.Contains(userContent, "the document content") {- t.Errorf("prompt missing chunk text: %q", userContent)- }- // Regression guard: when the prompt embeds {text}, the chunk text- // must appear exactly once — buildExtractorMessages must not append- // it a second time (placeholder duplication bug).- if n := strings.Count(userContent, "the document content"); n != 1 {- t.Errorf("chunk text appears %d times, want 1: %q", n, userContent)- }-}--// TestExtractorComponent_Invoke_PlaceholderChunksAlias verifies that-// {chunks} (the Python DSL upstream key) is also substituted with-// the current chunk text.-func TestExtractorComponent_Invoke_PlaceholderChunksAlias(t *testing.T) {- stub := withStubChatInvoker(t,- stubResponse{Content: "answer"},- )-- c := &ExtractorComponent{Param: schema.ExtractorParam{- FieldName: "out",- Prompt: "Content: {chunks}",- LLMID: "gpt-4o-mini",- }}- _, err := c.Invoke(t.Context(), nil, map[string]any{- "chunks": []map[string]any{{"content_with_weight": "weighted doc"}},- })- if err != nil {- t.Fatalf("Invoke: %v", err)- }-- stub.mu.Lock()- defer stub.mu.Unlock()- var userContent string- for _, msg := range stub.lastRequest().Messages {- if msg.Role == eschema.User {- userContent = msg.Content- }- }- if strings.Contains(userContent, "{chunks}") {- t.Errorf("prompt still contains literal {chunks}: %q", userContent)- }- if !strings.Contains(userContent, "weighted doc") {- t.Errorf("prompt missing chunk text: %q", userContent)- }- // Regression guard: {chunks} must not duplicate the chunk text.- if n := strings.Count(userContent, "weighted doc"); n != 1 {- t.Errorf("chunk text appears %d times, want 1: %q", n, userContent)- }-}--// TestExtractorComponent_Invoke_AppendsChunkTextWhenNoPlaceholder verifies-// that when the prompt has no {text}/{chunks} placeholder, the chunk text is-// still automatically appended by buildExtractorMessages exactly once.-func TestExtractorComponent_Invoke_AppendsChunkTextWhenNoPlaceholder(t *testing.T) {- stub := withStubChatInvoker(t,- stubResponse{Content: "answer"},- )-- c := &ExtractorComponent{Param: schema.ExtractorParam{- FieldName: "summary",- Prompt: "Summarize the above:",- LLMID: "gpt-4o-mini",- }}- _, err := c.Invoke(t.Context(), nil, map[string]any{- "chunks": []map[string]any{{"text": "the document content"}},- })- if err != nil {- t.Fatalf("Invoke: %v", err)- }-- stub.mu.Lock()- defer stub.mu.Unlock()- var userContent string- for _, msg := range stub.lastRequest().Messages {- if msg.Role == eschema.User {- userContent = msg.Content- }- }- if n := strings.Count(userContent, "the document content"); n != 1 {- t.Errorf("chunk text appears %d times, want 1: %q", n, userContent)- }-}--// TestExtractorComponent_Invoke_SystemPromptPlaceholderSuppressesAppend-// verifies that a content-bearing placeholder in systemPrompt (not just-// prompt) also suppresses the automatic chunk-text append. The chunk body-// is delivered via systemPrompt substitution; since the append is also-// suppressed, it must NOT appear a second time in the user message.-func TestExtractorComponent_Invoke_SystemPromptPlaceholderSuppressesAppend(t *testing.T) {- stub := withStubChatInvoker(t,- stubResponse{Content: "answer"},- )-- c := &ExtractorComponent{Param: schema.ExtractorParam{- FieldName: "out",- Prompt: "Extract:",- SystemPrompt: "Context: {text}",- LLMID: "gpt-4o-mini",- }}- _, err := c.Invoke(t.Context(), nil, map[string]any{- "chunks": []map[string]any{{"text": "system prompt body"}},- })- if err != nil {- t.Fatalf("Invoke: %v", err)- }-- stub.mu.Lock()- defer stub.mu.Unlock()- var sysContent, userContent string- for _, msg := range stub.lastRequest().Messages {- switch msg.Role {- case eschema.System:- sysContent = msg.Content- case eschema.User:- userContent = msg.Content- }- }- // {text} in systemPrompt must be resolved to the chunk body.- if !strings.Contains(sysContent, "system prompt body") {- t.Errorf("system message missing chunk body: %q", sysContent)- }- // The append must be suppressed: the user message should NOT also- // contain the chunk body (otherwise it is duplicated).- if strings.Contains(userContent, "system prompt body") {- t.Errorf("chunk body duplicated into user message (append not suppressed): %q", userContent)- }-}--// TestExtractorComponent_Invoke_FieldValueContainsPlaceholderSubstring-// verifies that a chunk field whose value happens to contain a content-// placeholder substring (e.g. title = "{text}") does not fool the-// suppression check. {text} in prompt is resolved to "body"; {title}-// is resolved to "{text}" literally — the substitution function knows-// {text} was actually replaced (title's replacement is a different-// placeholder), so suppression triggers correctly.-func TestExtractorComponent_Invoke_FieldValueContainsPlaceholderSubstring(t *testing.T) {- stub := withStubChatInvoker(t,- stubResponse{Content: "answer"},- )-- c := &ExtractorComponent{Param: schema.ExtractorParam{- FieldName: "out",- Prompt: "Body: {text}\nLabel: {title}",- LLMID: "gpt-4o-mini",- }}- _, err := c.Invoke(t.Context(), nil, map[string]any{- "chunks": []map[string]any{{- "text": "the document body",- "title": "{text}",- }},- })- if err != nil {- t.Fatalf("Invoke: %v", err)- }-- stub.mu.Lock()- defer stub.mu.Unlock()- var userContent string- for _, msg := range stub.lastRequest().Messages {- if msg.Role == eschema.User {- userContent = msg.Content- }- }- // {text} resolved to body → append suppressed. The body should appear- // exactly once (from {text} substitution), not twice.- if n := strings.Count(userContent, "the document body"); n != 1 {- t.Errorf("chunk text appears %d times, want 1 (no duplicate append): %q", n, userContent)- }- // {title} was substituted to the literal "{text}" — this is the tricky- // case: the substituted prompt now contains "{text}" as a value, but- // the suppression must still have triggered because {text} was resolved.- if !strings.Contains(userContent, "Label: {text}") {- t.Errorf("expected title substitution to produce literal '{text}' label: %q", userContent)- }-}--// TestFitExtractorMessages_RejectsEmptyUserTurn verifies that when-// messagefit's proportional trim would empty the final user turn (the system-// prompt alone exceeds the context budget), the extractor surfaces a clear-// error instead of sending [system, user:""] to the provider.-func TestFitExtractorMessages_RejectsEmptyUserTurn(t *testing.T) {- SetExtractorContextLengthOverride(func(_ context.Context, _ string) int { return 500 })- t.Cleanup(func() { SetExtractorContextLengthOverride(nil) })-- msgs := []eschema.Message{- {Role: eschema.System, Content: strings.Repeat("s ", 1000)},- {Role: eschema.User, Content: strings.Repeat("u ", 400)},- }- if _, err := fitExtractorMessages(t.Context(), nil, "test@test", msgs); err == nil {- t.Fatal("expected an error when fitting empties the user turn")- }-}--// TestFitExtractorMessages_KeepsUserTurn verifies the happy path: with a-// normal budget the fitter trims oversized prompts and the final user turn-// survives, so no error is returned.-func TestFitExtractorMessages_KeepsUserTurn(t *testing.T) {- SetExtractorContextLengthOverride(func(_ context.Context, _ string) int { return 2000 })- t.Cleanup(func() { SetExtractorContextLengthOverride(nil) })-- msgs := []eschema.Message{- {Role: eschema.System, Content: "you are a helpful assistant"},- {Role: eschema.User, Content: strings.Repeat("u ", 3000)},- }- fitted, err := fitExtractorMessages(t.Context(), nil, "test@test", msgs)- if err != nil {- t.Fatalf("unexpected error: %v", err)- }- if len(fitted) != 2 {- t.Fatalf("got %d messages, want 2", len(fitted))- }- if strings.TrimSpace(fitted[1].Content) == "" {- t.Fatal("user turn was emptied")- }-}--// TestFitExtractorMessages_NoSystemPromptKeepsUserTurn verifies that a-// user-only request (no system prompt configured) is not rejected by the-// system-prompt guard: the guard only applies when a system message was-// actually present, so a valid prompt-only extractor keeps working once the-// model's content_length is resolvable.-func TestFitExtractorMessages_NoSystemPromptKeepsUserTurn(t *testing.T) {- SetExtractorContextLengthOverride(func(_ context.Context, _ string) int { return 2000 })- t.Cleanup(func() { SetExtractorContextLengthOverride(nil) })-- msgs := []eschema.Message{- {Role: eschema.User, Content: strings.Repeat("u ", 3000)},- }- fitted, err := fitExtractorMessages(t.Context(), nil, "test@test", msgs)- if err != nil {- t.Fatalf("unexpected error for user-only prompt: %v", err)- }- if len(fitted) != 1 || fitted[0].Role != eschema.User {- t.Fatalf("got %d messages, want the single user turn: %+v", len(fitted), fitted)- }- if strings.TrimSpace(fitted[0].Content) == "" {- t.Fatal("user turn was emptied")- }-}--// TestExtractorComponent_CallRaw_FitsBeforeInvoke verifies the production-// wiring end to end: callRaw resolves the model's context length, trims the-// messages to the budget, and hands the fitted messages to the invoker.-func TestExtractorComponent_CallRaw_FitsBeforeInvoke(t *testing.T) {- SetExtractorContextLengthOverride(func(_ context.Context, _ string) int { return 200 })- t.Cleanup(func() { SetExtractorContextLengthOverride(nil) })-- stub := withStubChatInvoker(t, stubResponse{Content: `{"ok": true}`})- c := &ExtractorComponent{}-- // callRaw is a pure dispatcher: callers are responsible for pre-rendering- // the prompt. Inline a large chunk body directly into the user prompt so- // fitExtractorMessages has real content to trim.- chunkBody := strings.Repeat("chunk text with lots of tokens. ", 500)- _, err := c.callText(t.Context(), nil, extractorInputs{- systemPrompt: "extract fields",- prompt: "summarize\n\n" + chunkBody,- llmID: "test@test",- }, "")- if err != nil {- t.Fatalf("callText: %v", err)- }-- stub.mu.Lock()- req := stub.lastRequest()- stub.mu.Unlock()- if len(req.Messages) == 0 {- t.Fatal("invoker was not called")- }- if req.Messages[0].Role != eschema.System || strings.TrimSpace(req.Messages[0].Content) == "" {- t.Fatalf("system prompt lost or emptied before invoke: %+v", req.Messages[0])- }- total := 0- for _, m := range req.Messages {- total += tokenizer.NumTokensFromString(m.Content)+ total := 0+ for _, m := range req.Messages {+ total += tokenizer.NumTokensFromString(m.Content)}if total > extractorContextFitBudget(200) {t.Fatalf("sent messages total %d exceed the fitting budget %d", total, extractorContextFitBudget(200))@@ -1907,15 +1139,10 @@ func TestExtractorComponent_CallRaw_FitsBeforeInvoke(t *testing.T) {}}-// TestExtractorComponent_CallRaw_CustomContextOverride verifies the extractor-// wiring honors the tenant-configured override end to end: with a 2000-token-// extra max_tokens on the gpt-4o row, the invoker receives messages fitted to-// ~1940 tokens instead of the catalog's 128k.+// TestExtractorComponent_CallRaw_CustomContextOverride verifies tenant-configured context override.func TestExtractorComponent_CallRaw_CustomContextOverride(t *testing.T) {db := openExtractorContextTestDB(t)seedExtractorContextModel(t, db, "")- // Add the instance row the composite resolution path needs, then pin the- // tenant-configured context override on the model.if err := db.Create(&entity.TenantModelInstance{ID: "instance-1",ProviderID: "provider-openai",@@ -1934,10 +1161,8 @@ func TestExtractorComponent_CallRaw_CustomContextOverride(t *testing.T) {stub := withStubChatInvoker(t, stubResponse{Content: `{"ok": true}`})c := &ExtractorComponent{}_, err := c.callText(ctx, db, extractorInputs{- systemPrompt: "extract fields",- prompt: "summarize",- llmID: "gpt-4o@OpenAI",- }, strings.Repeat("chunk text with lots of tokens. ", 500))+ llmID: "gpt-4o@OpenAI",+ }, "extract fields", strings.Repeat("chunk text with lots of tokens. ", 500))if err != nil {t.Fatalf("callText: %v", err)}@@ -1960,10 +1185,6 @@ func TestExtractorComponent_CallRaw_CustomContextOverride(t *testing.T) {}}-// openExtractorContextTestDB returns an in-memory DB with the tenant and-// tenant-model tables migrated. Tests pass the returned handle explicitly to-// extractorContextLength, defaultChatModelRef, and dao.ResolveModelContentLength,-// so no global DAO state is touched.func openExtractorContextTestDB(t *testing.T) *gorm.DB {t.Helper()db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{TranslateError: true})@@ -1976,10 +1197,6 @@ func openExtractorContextTestDB(t *testing.T) *gorm.DB {return db}-// seedExtractorContextModel seeds an active OpenAI gpt-4o tenant model-// (catalog content_length 128000) plus its tenant. tenantLLMID, when-// non-empty, pins the tenant's default chat model to the tenant-model UUID;-// otherwise the tenant falls back to the composite llm_id.func seedExtractorContextModel(t *testing.T, db *gorm.DB, tenantLLMID string) {t.Helper()status := "1"@@ -2013,8 +1230,6 @@ func seedExtractorContextModel(t *testing.T, db *gorm.DB, tenantLLMID string) {}}-// extractorStateCtx returns a context carrying a canvas state with the given-// tenant_id global, as extractorContextLength expects.func extractorStateCtx(t *testing.T, tenantID string) context.Context {t.Helper()state := runtime.NewCanvasState("run-1", "session-1")@@ -2022,9 +1237,6 @@ func extractorStateCtx(t *testing.T, tenantID string) context.Context {return runtime.WithState(t.Context(), state)}-// TestExtractorContextLength_TenantModelUUID verifies extractorContextLength-// resolves content_length for a tenant_model UUID through the provider-// catalog.func TestExtractorContextLength_TenantModelUUID(t *testing.T) {db := openExtractorContextTestDB(t)seedExtractorContextModel(t, db, "")@@ -2035,9 +1247,6 @@ func TestExtractorContextLength_TenantModelUUID(t *testing.T) {}}-// TestExtractorContextLength_DefaultChatModelPinned verifies the llmID==""-// fallback resolves the tenant default chat model when it is pinned to a-// tenant_model UUID.func TestExtractorContextLength_DefaultChatModelPinned(t *testing.T) {db := openExtractorContextTestDB(t)seedExtractorContextModel(t, db, "0123456789abcdef0123456789abcdef")@@ -2048,9 +1257,6 @@ func TestExtractorContextLength_DefaultChatModelPinned(t *testing.T) {}}-// TestExtractorContextLength_DefaultChatModelComposite verifies the llmID==""-// fallback resolves the tenant default chat model from the composite-// "model@provider" llm_id when no tenant_model is pinned.func TestExtractorContextLength_DefaultChatModelComposite(t *testing.T) {db := openExtractorContextTestDB(t)seedExtractorContextModel(t, db, "")@@ -2061,8 +1267,6 @@ func TestExtractorContextLength_DefaultChatModelComposite(t *testing.T) {}}-// TestExtractorContextLength_UnknownModelSkips verifies extractorContextLength-// returns 0 (skip fitting) for an unknown model reference.func TestExtractorContextLength_UnknownModelSkips(t *testing.T) {db := openExtractorContextTestDB(t)seedExtractorContextModel(t, db, "")@@ -2073,31 +1277,19 @@ func TestExtractorContextLength_UnknownModelSkips(t *testing.T) {}}-// TestExtractorContextFitBudget verifies the fitting budget is 97% of the-// resolved content_length (mirroring the agent's contextFitBudget), leaving-// headroom for tokenizer drift between cl100k and the model's own tokenizer,-// and that a tiny context never collapses to messagefit's <=0 → 8192 default.func TestExtractorContextFitBudget(t *testing.T) {if got := extractorContextFitBudget(128000); got != 124160 {t.Fatalf("extractorContextFitBudget(128000) = %d, want 124160", got)}if got := extractorContextFitBudget(1); got != 1 {- t.Fatalf("extractorContextFitBudget(1) = %d, want 1 (clamped to avoid the 8192 Fit default)", got)+ t.Fatalf("extractorContextFitBudget(1) = %d, want 1", got)}}-// TestFitExtractorMessages_RejectsSystemPromptLoss verifies the guard that a-// fitting which empties every system message is rejected instead of sending-// an instruction-less extraction request: the system prompt carries the-// extraction contract, so running with an emptied system prompt would-// silently produce garbage.func TestFitExtractorMessages_RejectsSystemPromptLoss(t *testing.T) {SetExtractorContextLengthOverride(func(_ context.Context, _ string) int { return 300 })t.Cleanup(func() { SetExtractorContextLengthOverride(nil) })- // System dominates (>4x the user) and the user message alone exceeds- // the budget: the proportional trim preserves the user turn and empties- // the system messages.msgs := []eschema.Message{{Role: eschema.System, Content: strings.Repeat("s ", 5000)},{Role: eschema.User, Content: strings.Repeat("u ", 400)},@@ -2107,226 +1299,41 @@ func TestFitExtractorMessages_RejectsSystemPromptLoss(t *testing.T) {}}-// TestExtractorContextLength_NilDBGraceful verifies that resolving the tenant-// default chat model with no database available (nil db and no override)-// degrades to 0 (skip fitting) instead of panicking in defaultChatModelRef.func TestExtractorContextLength_NilDBGraceful(t *testing.T) {ctx := extractorStateCtx(t, "tenant-1")if got := extractorContextLength(ctx, nil, ""); got != 0 {- t.Fatalf("extractorContextLength(nil db, default model) = %d, want 0 (skip fitting)", got)+ t.Fatalf("extractorContextLength(nil db, default model) = %d, want 0", got)}}-// TestExtractorComponent_Invoke_AtChunksPlaceholderPerChunk verifies that-// when the prompt contains {ComponentName:ParamName@chunks}, each chunk's-// LLM invocation receives ONLY that chunk's text (not all chunks joined together),-// and the chunk text appears exactly once without duplication.-func TestExtractorComponent_Invoke_AtChunksPlaceholderPerChunk(t *testing.T) {- stub := withStubChatInvoker(t,- stubResponse{Content: "answer1"},- stubResponse{Content: "answer2"},- )-- c := &ExtractorComponent{Param: schema.ExtractorParam{- FieldName: "summary",- Prompt: "Summarize: {TokenChunker:BumpyStarsPress@chunks}",- LLMID: "gpt-4o-mini",- }}-- _, err := c.Invoke(t.Context(), nil, map[string]any{- "chunks": []map[string]any{- {"text": "Chunk One Body"},- {"text": "Chunk Two Body"},- },- })- if err != nil {- t.Fatalf("Invoke: %v", err)- }-- stub.mu.Lock()- defer stub.mu.Unlock()-- if len(stub.requests) != 2 {- t.Fatalf("got %d LLM calls, want 2", len(stub.requests))+func TestBuildExtractorMessages(t *testing.T) {+ msgs := buildExtractorMessages("System rule", "Chunk content")+ if len(msgs) != 2 {+ t.Fatalf("expected 2 messages, got %d", len(msgs))}-- // Call 1 must contain Chunk 1 exactly once, and NOT contain Chunk 2- req1User := stub.requests[0].Messages[len(stub.requests[0].Messages)-1].Content- if n := strings.Count(req1User, "Chunk One Body"); n != 1 {- t.Errorf("call 1 chunk text count = %d, want 1; prompt: %q", n, req1User)+ if msgs[0].Role != eschema.System || msgs[0].Content != "System rule" {+ t.Errorf("msgs[0] = %#v, want system message", msgs[0])}- if strings.Contains(req1User, "Chunk Two Body") {- t.Errorf("call 1 wrongly contains Chunk Two Body; prompt: %q", req1User)+ if msgs[1].Role != eschema.User || msgs[1].Content != "Chunk content" {+ t.Errorf("msgs[1] = %#v, want user message", msgs[1])}- // Call 2 must contain Chunk 2 exactly once, and NOT contain Chunk 1- req2User := stub.requests[1].Messages[len(stub.requests[1].Messages)-1].Content- if n := strings.Count(req2User, "Chunk Two Body"); n != 1 {- t.Errorf("call 2 chunk text count = %d, want 1; prompt: %q", n, req2User)+ // Empty system prompt omitted+ msgsNoSys := buildExtractorMessages("", "Chunk content")+ if len(msgsNoSys) != 1 {+ t.Fatalf("expected 1 message, got %d", len(msgsNoSys))}- if strings.Contains(req2User, "Chunk One Body") {- t.Errorf("call 2 wrongly contains Chunk One Body; prompt: %q", req2User)+ if msgsNoSys[0].Role != eschema.User || msgsNoSys[0].Content != "Chunk content" {+ t.Errorf("msgsNoSys[0] = %#v, want user message", msgsNoSys[0])}-}-// TestRenderExtractorPrompts_TableDriven covers all prompt rendering cases-// and fallback permutations.-func TestRenderExtractorPrompts_TableDriven(t *testing.T) {- tests := []struct {- name string- sysTemplate string- userTemplate string- ck map[string]any- chunkText string- wantSys string- wantUser string- }{- {- name: "text in user",- sysTemplate: "",- userTemplate: "Analyze: {text}",- ck: map[string]any{"text": "body content"},- chunkText: "body content",- wantSys: "",- wantUser: "Analyze: body content",- },- {- name: "text in system only suppresses user append",- sysTemplate: "Context: {text}",- userTemplate: "Extract:",- ck: map[string]any{"text": "body content"},- chunkText: "body content",- wantSys: "Context: body content",- wantUser: "Extract:",- },- {- name: "canvas macro @chunks in user",- sysTemplate: "",- userTemplate: "Summarize: {TokenChunker:BumpyStarsPress@chunks}",- ck: map[string]any{"text": "body content"},- chunkText: "body content",- wantSys: "",- wantUser: "Summarize: body content",- },- {- name: "canvas macro @text in user",- sysTemplate: "",- userTemplate: "Parse: {Parser:Doc@text}",- ck: map[string]any{"text": "body content"},- chunkText: "body content",- wantSys: "",- wantUser: "Parse: body content",- },- {- name: "canvas macro @markdown in user",- sysTemplate: "",- userTemplate: "Parse: {Parser:Doc@markdown}",- ck: map[string]any{"text": "body content"},- chunkText: "body content",- wantSys: "",- wantUser: "Parse: body content",- },- {- name: "metadata only appends chunkText",- sysTemplate: "",- userTemplate: "Title: {title}",- ck: map[string]any{"title": "DocTitle", "text": "body content"},- chunkText: "body content",- wantSys: "",- wantUser: "Title: DocTitle\n\nbody content",- },- {- name: "no placeholder appends chunkText",- sysTemplate: "",- userTemplate: "Summarize the text:",- ck: map[string]any{"text": "body content"},- chunkText: "body content",- wantSys: "",- wantUser: "Summarize the text:\n\nbody content",- },- {- name: "empty chunkText does not append",- sysTemplate: "",- userTemplate: "Summarize:",- ck: map[string]any{},- chunkText: "",- wantSys: "",- wantUser: "Summarize:",- },- {- name: "content_with_weight present in ck",- sysTemplate: "",- userTemplate: "Weighted: {content_with_weight}",- ck: map[string]any{"content_with_weight": "weighted content", "text": "plain content"},- chunkText: "weighted content",- wantSys: "",- wantUser: "Weighted: weighted content",- },- {- name: "content_with_weight absent in ck falls back to chunkText",- sysTemplate: "",- userTemplate: "Weighted: {content_with_weight}",- ck: map[string]any{"text": "plain content"},- chunkText: "plain content",- wantSys: "",- wantUser: "Weighted: plain content",- },- {- name: "whitespace user template trimmed before append",- sysTemplate: "",- userTemplate: " ",- ck: map[string]any{"text": "body content"},- chunkText: "body content",- wantSys: "",- wantUser: "body content",- },- {- name: "empty templates produce single space user turn",- sysTemplate: "",- userTemplate: "",- ck: map[string]any{},- chunkText: "",- wantSys: "",- wantUser: " ",- },- {- // content_with_weight is present in ck but its value is an empty- // string. isBodyPlaceholder fires, but the resolved value is empty- // so bodyInjected must NOT be set — the fallback append should- // still fire and deliver chunkText.- name: "content_with_weight empty string in ck falls back to chunkText",- sysTemplate: "",- userTemplate: "Weighted: {content_with_weight}",- ck: map[string]any{"content_with_weight": ""},- chunkText: "fallback body",- wantSys: "",- wantUser: "Weighted: fallback body",- },- {- // Both system and user prompts reference body placeholders.- // The body must appear in each substitution position but must NOT- // be appended a third time (bodyInjected is shared across both- // render calls).- name: "body placeholder in both system and user no extra append",- sysTemplate: "Context: {text}",- userTemplate: "Also: {chunks}",- ck: map[string]any{"text": "the body"},- chunkText: "the body",- wantSys: "Context: the body",- wantUser: "Also: the body",- },+ // Empty user chunk text normalized to single space+ msgsEmptyUser := buildExtractorMessages("System rule", "")+ if len(msgsEmptyUser) != 2 {+ t.Fatalf("expected 2 messages, got %d", len(msgsEmptyUser))}-- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- gotSys, gotUser := renderExtractorPrompts(tt.sysTemplate, tt.userTemplate, tt.ck, tt.chunkText)- if gotSys != tt.wantSys {- t.Errorf("renderExtractorPrompts() gotSys = %q, want %q", gotSys, tt.wantSys)- }- if gotUser != tt.wantUser {- t.Errorf("renderExtractorPrompts() gotUser = %q, want %q", gotUser, tt.wantUser)- }- })+ if msgsEmptyUser[1].Role != eschema.User || msgsEmptyUser[1].Content != " " {+ t.Errorf("msgsEmptyUser[1] = %#v, want single space", msgsEmptyUser[1])}}@@ -2334,20 +1341,23 @@ func TestExtractorModularParams(t *testing.T) {params := map[string]any{"llm_id": "test-llm-1","keywords": map[string]any{- "top_n": 5,- "system_prompt": "Custom keywords prompt for {text}",+ "top_n": 5,},"questions": map[string]any{- "top_n": 3,- "system_prompt": "Custom questions prompt for {text}",+ "top_n": 3,},"tags": map[string]any{"top_n": 2,"tag_file_id": "tag_file_abc",},"summary": map[string]any{- "enabled": true,- "system_prompt": "Custom summary prompt for {text}",+ "enabled": true,+ },+ "metadata": map[string]any{+ "enabled": true,+ "metadata": []any{+ map[string]any{"key": "category", "type": "string"},+ },},}@@ -2361,67 +1371,20 @@ func TestExtractorModularParams(t *testing.T) {t.Fatalf("expected *ExtractorComponent, got %T", comp)}- if ext.Param.Keywords.TopN != 5 || ext.Param.Keywords.SystemPrompt != "Custom keywords prompt for {text}" {+ if ext.Param.Keywords.TopN != 5 {t.Errorf("Keywords config mismatch: %+v", ext.Param.Keywords)}- if ext.Param.AutoKeywords != 5 {- t.Errorf("AutoKeywords flat sync mismatch: %d", ext.Param.AutoKeywords)- }-- if ext.Param.Questions.TopN != 3 || ext.Param.Questions.SystemPrompt != "Custom questions prompt for {text}" {+ if ext.Param.Questions.TopN != 3 {t.Errorf("Questions config mismatch: %+v", ext.Param.Questions)}- if ext.Param.AutoQuestions != 3 {- t.Errorf("AutoQuestions flat sync mismatch: %d", ext.Param.AutoQuestions)- }-if ext.Param.Tags.TopN != 2 || ext.Param.Tags.TagFileID != "tag_file_abc" {t.Errorf("Tags config mismatch: %+v", ext.Param.Tags)}- if ext.Param.AutoTags != 2 || ext.Param.TagFileID != "tag_file_abc" {- t.Errorf("AutoTags flat sync mismatch: %d, %s", ext.Param.AutoTags, ext.Param.TagFileID)- }-- if !ext.Param.Summary.Enabled || ext.Param.Summary.SystemPrompt != "Custom summary prompt for {text}" {+ if !ext.Param.Summary.Enabled {t.Errorf("Summary config mismatch: %+v", ext.Param.Summary)}- if ext.Param.FieldName != "summary" {- t.Errorf("FieldName mismatch: %s", ext.Param.FieldName)- }-}--func TestExtractorLegacyFlatParamsFallback(t *testing.T) {- params := map[string]any{- "llm_id": "test-llm-1",- "auto_keywords": 4,- "auto_questions": 2,- "auto_tags": 1,- "tag_file_id": "tag_file_legacy",- "field_name": "summary",- "sys_prompt": "Legacy summary prompt",- }-- comp, err := NewExtractorComponent(params)- if err != nil {- t.Fatalf("NewExtractorComponent failed: %v", err)- }-- ext, ok := comp.(*ExtractorComponent)- if !ok {- t.Fatalf("expected *ExtractorComponent, got %T", comp)- }-- if ext.Param.Keywords.TopN != 4 || ext.Param.AutoKeywords != 4 {- t.Errorf("Keywords fallback mismatch: %d / %d", ext.Param.Keywords.TopN, ext.Param.AutoKeywords)- }- if ext.Param.Questions.TopN != 2 || ext.Param.AutoQuestions != 2 {- t.Errorf("Questions fallback mismatch: %d / %d", ext.Param.Questions.TopN, ext.Param.AutoQuestions)- }- if ext.Param.Tags.TopN != 1 || ext.Param.Tags.TagFileID != "tag_file_legacy" {- t.Errorf("Tags fallback mismatch: %+v", ext.Param.Tags)- }- if !ext.Param.Summary.Enabled || ext.Param.Summary.SystemPrompt != "Legacy summary prompt" {- t.Errorf("Summary fallback mismatch: %+v", ext.Param.Summary)+ if !ext.Param.Metadata.Enabled || len(ext.Param.Metadata.Metadata) != 1 {+ t.Errorf("Metadata config mismatch: %+v", ext.Param.Metadata)}}@@ -2435,16 +1398,13 @@ func TestExtractorModularPromptsExecution(t *testing.T) {params := map[string]any{"llm_id": "llm-1","keywords": map[string]any{- "top_n": 2,- "system_prompt": "Custom KW Prompt: {text}",+ "top_n": 2,},"questions": map[string]any{- "top_n": 2,- "system_prompt": "Custom Q Prompt: {text}",+ "top_n": 2,},"summary": map[string]any{- "enabled": true,- "system_prompt": "Custom Sum Prompt: {text}",+ "enabled": true,},}@@ -2480,42 +1440,16 @@ func TestExtractorModularPromptsExecution(t *testing.T) {t.Errorf("expected summary = 'Summary of chunk', got %v", ck["summary"])}- // Verify custom prompts were rendered into messages and passed to the chat invokerreqs := stub.requestsif len(reqs) != 3 {t.Fatalf("expected 3 LLM calls, got %d", len(reqs))}-- var foundKW, foundQ, foundSum bool- for _, r := range reqs {- for _, msg := range r.Messages {- if strings.Contains(msg.Content, "Custom KW Prompt: Hello world content") {- foundKW = true- }- if strings.Contains(msg.Content, "Custom Q Prompt: Hello world content") {- foundQ = true- }- if strings.Contains(msg.Content, "Custom Sum Prompt: Hello world content") {- foundSum = true- }- }- }-- if !foundKW {- t.Errorf("expected custom keyword prompt to be used in LLM call, got reqs: %+v", reqs)- }- if !foundQ {- t.Errorf("expected custom question prompt to be used in LLM call, got reqs: %+v", reqs)- }- if !foundSum {- t.Errorf("expected custom summary prompt to be used in LLM call, got reqs: %+v", reqs)- }}func TestExtractorModularMetadataConfig(t *testing.T) {params := map[string]any{"llm_id": "llm-1",- "metadata_config": map[string]any{+ "metadata": map[string]any{"enabled": true,"metadata": []any{map[string]any{@@ -2548,17 +1482,17 @@ func TestExtractorModularMetadataConfig(t *testing.T) {t.Fatalf("expected *ExtractorComponent, got %T", comp)}- if !ext.Param.MetadataConfig.Enabled || ext.Param.EnableMetadata != 1 {- t.Errorf("Metadata enabled mismatch: %+v / %d", ext.Param.MetadataConfig.Enabled, ext.Param.EnableMetadata)+ if !ext.Param.Metadata.Enabled {+ t.Errorf("Metadata enabled mismatch: %+v", ext.Param.Metadata.Enabled)}- if len(ext.Param.MetadataConfig.Metadata) != 2 || len(ext.Param.Metadata) != 2 {- t.Fatalf("Metadata fields mismatch: %d / %d", len(ext.Param.MetadataConfig.Metadata), len(ext.Param.Metadata))+ if len(ext.Param.Metadata.Metadata) != 2 {+ t.Fatalf("Metadata fields mismatch: %d", len(ext.Param.Metadata.Metadata))}- if ext.Param.Metadata[0].Key != "author" || ext.Param.Metadata[0].Description != "The author name" || len(ext.Param.Metadata[0].Enum) != 2 {- t.Errorf("Metadata field 0 mismatch: %+v", ext.Param.Metadata[0])+ if ext.Param.Metadata.Metadata[0].Key != "author" || ext.Param.Metadata.Metadata[0].Description != "The author name" || len(ext.Param.Metadata.Metadata[0].Enum) != 2 {+ t.Errorf("Metadata field 0 mismatch: %+v", ext.Param.Metadata.Metadata[0])}- if len(ext.Param.MetadataConfig.BuiltInMetadata) != 1 || ext.Param.MetadataConfig.BuiltInMetadata[0].Key != "file_name" {- t.Errorf("BuiltInMetadata mismatch: %+v", ext.Param.MetadataConfig.BuiltInMetadata)+ if len(ext.Param.Metadata.BuiltInMetadata) != 1 || ext.Param.Metadata.BuiltInMetadata[0].Key != "file_name" {+ t.Errorf("BuiltInMetadata mismatch: %+v", ext.Param.Metadata.BuiltInMetadata)}}@@ -2569,7 +1503,7 @@ func TestExtractorModularMetadataExecution(t *testing.T) {params := map[string]any{"llm_id": "llm-1",- "metadata_config": map[string]any{+ "metadata": map[string]any{"enabled": true,"metadata": []any{map[string]any{@@ -2613,7 +1547,6 @@ func TestExtractorModularMetadataExecution(t *testing.T) {if meta["author"] != "Alice" {t.Errorf("expected metadata.author = Alice, got %v", meta["author"])}- // year can be float64 or int from JSONif fmt.Sprintf("%v", meta["year"]) != "2026" {t.Errorf("expected metadata.year = 2026, got %v", meta["year"])}@@ -2626,7 +1559,6 @@ func TestExtractorDefaultSummaryPromptInjection(t *testing.T) {"llm_id": "llm-1","summary": map[string]any{"enabled": true,- // system_prompt left empty to test default prompt injection},}@@ -2674,6 +1606,109 @@ func TestExtractorDefaultSummaryPromptInjection(t *testing.T) {}}+func TestExtractorCustomSummarySystemPrompt(t *testing.T) {+ stub := withStubChatInvoker(t, stubResponse{Content: "Custom summary."})++ params := map[string]any{+ "llm_id": "llm-1",+ "summary": map[string]any{+ "enabled": true,+ "system_prompt": "Custom system prompt for summarization.",+ },+ }++ comp, err := NewExtractorComponent(params)+ if err != nil {+ t.Fatalf("NewExtractorComponent: %v", err)+ }++ in := map[string]any{+ "chunks": []map[string]any{+ {"content_with_weight": "Text to summarize."},+ },+ }++ out, err := comp.Invoke(t.Context(), nil, in)+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }++ chunks, ok := out["chunks"].([]map[string]any)+ if !ok || len(chunks) != 1 {+ t.Fatalf("expected 1 chunk output, got %v", out)+ }++ if chunks[0]["summary"] != "Custom summary." {+ t.Errorf("expected summary = 'Custom summary.', got %v", chunks[0]["summary"])+ }++ if stub.Calls() != 1 {+ t.Fatalf("expected 1 LLM call, got %d", stub.Calls())+ }++ lastReq := stub.lastRequest()+ msgs := lastReq.Messages+ if len(msgs) != 2 {+ t.Fatalf("expected 2 messages (system + user), got %d: %+v", len(msgs), msgs)+ }++ if msgs[0].Role != "system" || msgs[0].Content != "Custom system prompt for summarization." {+ t.Errorf("expected custom system prompt in system message, got: %+v", msgs[0])+ }+}++func TestExtractorCustomKeywordsAndQuestionsSystemPrompt(t *testing.T) {+ stub := withStubChatInvoker(t,+ stubResponse{Content: "custom, keywords"},+ stubResponse{Content: "Custom question 1?\nCustom question 2?"},+ )++ params := map[string]any{+ "llm_id": "llm-1",+ "keywords": map[string]any{+ "top_n": 2,+ "system_prompt": "Custom keywords system prompt.",+ },+ "questions": map[string]any{+ "top_n": 2,+ "system_prompt": "Custom questions system prompt.",+ },+ }++ comp, err := NewExtractorComponent(params)+ if err != nil {+ t.Fatalf("NewExtractorComponent: %v", err)+ }++ in := map[string]any{+ "chunks": []map[string]any{+ {"content_with_weight": "Content text."},+ },+ }++ out, err := comp.Invoke(t.Context(), nil, in)+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }++ chunks, ok := out["chunks"].([]map[string]any)+ if !ok || len(chunks) != 1 {+ t.Fatalf("expected 1 chunk output, got %v", out)+ }++ reqs := stub.requests+ if len(reqs) != 2 {+ t.Fatalf("expected 2 LLM calls, got %d", len(reqs))+ }++ if reqs[0].Messages[0].Content != "Custom keywords system prompt." {+ t.Errorf("expected custom keywords system prompt, got: %q", reqs[0].Messages[0].Content)+ }+ if reqs[1].Messages[0].Content != "Custom questions system prompt." {+ t.Errorf("expected custom questions system prompt, got: %q", reqs[1].Messages[0].Content)+ }+}+func TestExtractorDisabledSummarySkipsCall(t *testing.T) {stub := withStubChatInvoker(t, stubResponse{Content: "Not expected"})@@ -2682,7 +1717,6 @@ func TestExtractorDisabledSummarySkipsCall(t *testing.T) {"summary": map[string]any{"enabled": false,},- "field_name": "summary", // legacy default that should be ignored when enabled=false}comp, err := NewExtractorComponent(params)@@ -2714,33 +1748,26 @@ func TestExtractorDisabledSummarySkipsCall(t *testing.T) {}}-func TestExtractor_NestedPrecedenceOverFlat(t *testing.T) {- // Case A: Nested explicitly disabled (enabled=false) while legacy flat has enabled=1/5.- // Nested MUST win and disable the features.+func TestExtractor_ModularConfiguration(t *testing.T) {paramsDisabled := map[string]any{- "metadata_config": map[string]any{+ "metadata": map[string]any{"enabled": false,- "metadata": []any{+ "fields": []any{map[string]any{"key": "category", "type": "string"},},},- "enable_metadata": 1,"summary": map[string]any{"enabled": false,},- "enable_summary": 1,"keywords": map[string]any{"top_n": 0,},- "auto_keywords": 5,"questions": map[string]any{"top_n": 0,},- "auto_questions": 3,"tags": map[string]any{"top_n": 0,},- "auto_tags": 2,}compRawA, err := NewExtractorComponent(paramsDisabled)@@ -2748,49 +1775,46 @@ func TestExtractor_NestedPrecedenceOverFlat(t *testing.T) {t.Fatalf("NewExtractorComponent A: %v", err)}compA := compRawA.(*ExtractorComponent)- if compA.Param.MetadataConfig.Enabled != false || compA.Param.EnableMetadata != 0 {- t.Errorf("expected metadata disabled, got nested=%v flat=%v", compA.Param.MetadataConfig.Enabled, compA.Param.EnableMetadata)+ if compA.Param.Metadata.Enabled != false {+ t.Errorf("expected metadata disabled, got %v", compA.Param.Metadata.Enabled)}- if compA.Param.Summary.Enabled != false || compA.Param.EnableSummary != 0 {- t.Errorf("expected summary disabled, got nested=%v flat=%v", compA.Param.Summary.Enabled, compA.Param.EnableSummary)+ if compA.Param.Summary.Enabled != false {+ t.Errorf("expected summary disabled, got %v", compA.Param.Summary.Enabled)}- if compA.Param.Keywords.TopN != 0 || compA.Param.AutoKeywords != 0 {- t.Errorf("expected keywords disabled (0), got nested=%v flat=%v", compA.Param.Keywords.TopN, compA.Param.AutoKeywords)+ if compA.Param.Keywords.TopN != 0 {+ t.Errorf("expected keywords disabled (0), got %v", compA.Param.Keywords.TopN)}- if compA.Param.Questions.TopN != 0 || compA.Param.AutoQuestions != 0 {- t.Errorf("expected questions disabled (0), got nested=%v flat=%v", compA.Param.Questions.TopN, compA.Param.AutoQuestions)+ if compA.Param.Questions.TopN != 0 {+ t.Errorf("expected questions disabled (0), got %v", compA.Param.Questions.TopN)}- if compA.Param.Tags.TopN != 0 || compA.Param.AutoTags != 0 {- t.Errorf("expected tags disabled (0), got nested=%v flat=%v", compA.Param.Tags.TopN, compA.Param.AutoTags)+ if compA.Param.Tags.TopN != 0 {+ t.Errorf("expected tags disabled (0), got %v", compA.Param.Tags.TopN)}- // Case B: Nested explicitly enabled (enabled=true) while legacy flat has 0.- // Nested MUST win and enable the features.+ // Explicitly enabledparamsEnabled := map[string]any{- "metadata_config": map[string]any{+ "metadata": map[string]any{"enabled": true,"metadata": []any{map[string]any{"key": "author", "type": "string"},},},- "enable_metadata": 0,"summary": map[string]any{- "enabled": true,+ "enabled": true,+ "system_prompt": "Custom summary prompt",},- "enable_summary": 0,"keywords": map[string]any{- "top_n": 4,+ "top_n": 4,+ "system_prompt": "Custom keywords prompt",},- "auto_keywords": 0,"questions": map[string]any{- "top_n": 2,+ "top_n": 2,+ "system_prompt": "Custom questions prompt",},- "auto_questions": 0,"tags": map[string]any{"top_n": 3,"tag_file_id": "file-123",},- "auto_tags": 0,}compRawB, err := NewExtractorComponent(paramsEnabled)@@ -2798,19 +1822,40 @@ func TestExtractor_NestedPrecedenceOverFlat(t *testing.T) {t.Fatalf("NewExtractorComponent B: %v", err)}compB := compRawB.(*ExtractorComponent)- if compB.Param.MetadataConfig.Enabled != true || compB.Param.EnableMetadata != 1 {- t.Errorf("expected metadata enabled, got nested=%v flat=%v", compB.Param.MetadataConfig.Enabled, compB.Param.EnableMetadata)+ if compB.Param.Metadata.Enabled != true {+ t.Errorf("expected metadata enabled, got %v", compB.Param.Metadata.Enabled)+ }+ if len(compB.Param.Metadata.Metadata) != 1 || compB.Param.Metadata.Metadata[0].Key != "author" {+ t.Errorf("expected metadata fields with author, got %+v", compB.Param.Metadata.Metadata)+ }+ if compB.Param.Summary.Enabled != true || compB.Param.Summary.SystemPrompt != "Custom summary prompt" {+ t.Errorf("expected summary enabled with custom prompt, got %+v", compB.Param.Summary)+ }+ if compB.Param.Keywords.TopN != 4 || compB.Param.Keywords.SystemPrompt != "Custom keywords prompt" {+ t.Errorf("expected keywords 4 with custom prompt, got %+v", compB.Param.Keywords)}- if compB.Param.Summary.Enabled != true || compB.Param.EnableSummary != 1 {- t.Errorf("expected summary enabled, got nested=%v flat=%v", compB.Param.Summary.Enabled, compB.Param.EnableSummary)+ if compB.Param.Questions.TopN != 2 || compB.Param.Questions.SystemPrompt != "Custom questions prompt" {+ t.Errorf("expected questions 2 with custom prompt, got %+v", compB.Param.Questions)}- if compB.Param.Keywords.TopN != 4 || compB.Param.AutoKeywords != 4 {- t.Errorf("expected keywords 4, got nested=%v flat=%v", compB.Param.Keywords.TopN, compB.Param.AutoKeywords)+ if compB.Param.Tags.TopN != 3 || compB.Param.Tags.TagFileID != "file-123" {+ t.Errorf("expected tags 3 / file-123, got %+v", compB.Param.Tags)}- if compB.Param.Questions.TopN != 2 || compB.Param.AutoQuestions != 2 {- t.Errorf("expected questions 2, got nested=%v flat=%v", compB.Param.Questions.TopN, compB.Param.AutoQuestions)+}++func TestExtractor_ParseMetadataFieldDefs_MapSlice(t *testing.T) {+ inputMapSlice := []map[string]any{+ {"key": "author", "type": "string", "description": "Author name"},+ }+ defs := parseMetadataFieldDefs(inputMapSlice)+ if len(defs) != 1 || defs[0].Key != "author" || defs[0].Type != "string" || defs[0].Description != "Author name" {+ t.Errorf("parseMetadataFieldDefs failed for []map[string]any: %#v", defs)+ }++ inputDefs := []common.MetadataFieldDef{+ {Key: "tag", Type: "string"},}- if compB.Param.Tags.TopN != 3 || compB.Param.AutoTags != 3 {- t.Errorf("expected tags 3, got nested=%v flat=%v", compB.Param.Tags.TopN, compB.Param.AutoTags)+ directDefs := parseMetadataFieldDefs(inputDefs)+ if len(directDefs) != 1 || directDefs[0].Key != "tag" {+ t.Errorf("parseMetadataFieldDefs failed for []common.MetadataFieldDef: %#v", directDefs)}}diff --git a/internal/ingestion/component/schema/extractor.go b/internal/ingestion/component/schema/extractor.goindex 4b123e73bf..bb5dc4e08d 100644--- a/internal/ingestion/component/schema/extractor.go+++ b/internal/ingestion/component/schema/extractor.go@@ -76,104 +76,53 @@ func (ExtractorFromUpstream) Validate() error { return nil }// KeywordExtractConfig configures automatic keyword extraction.type KeywordExtractConfig struct {- TopN int `json:"top_n,omitempty"`+ TopN int `json:"top_n"`SystemPrompt string `json:"system_prompt,omitempty"`}// QuestionExtractConfig configures automatic question generation.type QuestionExtractConfig struct {- TopN int `json:"top_n,omitempty"`+ TopN int `json:"top_n"`SystemPrompt string `json:"system_prompt,omitempty"`}// TagExtractConfig configures automatic tag extraction.type TagExtractConfig struct {- TopN int `json:"top_n,omitempty"`+ TopN int `json:"top_n"`TagFileID string `json:"tag_file_id,omitempty"`}// SummaryExtractConfig configures summary / enhanced context extraction.type SummaryExtractConfig struct {- Enabled bool `json:"enabled,omitempty"`+ Enabled bool `json:"enabled"`SystemPrompt string `json:"system_prompt,omitempty"`}// MetadataExtractConfig configures structured metadata extraction.type MetadataExtractConfig struct {- Enabled bool `json:"enabled,omitempty"`+ Enabled bool `json:"enabled"`Metadata []common.MetadataFieldDef `json:"metadata,omitempty"`BuiltInMetadata []common.MetadataFieldDef `json:"built_in_metadata,omitempty"`}-// ExtractorParam is the static configuration for the Extractor-// component. Supports modular sub-configs (Keywords, Questions, Tags,-// Summary, Metadata) as well as legacy flat fields for backward compatibility.+// ExtractorParam is the static configuration for the Extractor component.+// Fully modularized into base settings and 5 sub-extraction tasks.type ExtractorParam struct {- // Modular sub-configs- Keywords KeywordExtractConfig `json:"keywords,omitempty"`- Questions QuestionExtractConfig `json:"questions,omitempty"`- Tags TagExtractConfig `json:"tags,omitempty"`- Summary SummaryExtractConfig `json:"summary,omitempty"`- MetadataConfig MetadataExtractConfig `json:"metadata_config,omitempty"`-- // FieldName is the chunk key the LLM extraction result is written- // to (Python: `self._param.field_name`). Optional — when empty,- // auto_keywords or auto_questions may still be used.- FieldName string `json:"field_name,omitempty"`-- // LLMID identifies the LLM model used for extraction. This is the- // agent-side LLMParam.llm_id; on the ingestion side it is- // resolved against the tenant's LLM provider registry.+ // Base settingsLLMID string `json:"llm_id,omitempty"`- // SystemPrompt is the optional system prompt override.- SystemPrompt string `json:"system_prompt,omitempty"`-- // Prompt is the user-side template passed to the LLM.- Prompt string `json:"prompt,omitempty"`-- // AutoKeywords enables automatic keyword extraction with a fixed- // prompt. The value determines the top-N count. Legacy flat field.- AutoKeywords int `json:"auto_keywords,omitempty"`-- // AutoQuestions enables automatic question generation with a fixed- // prompt. The value determines the top-N count. Legacy flat field.- AutoQuestions int `json:"auto_questions,omitempty"`-- // AutoTags enables tag assignment on chunks. Legacy flat field.- AutoTags int `json:"auto_tags,omitempty"`-- // TagFileID references a tag-definition file stored in object- // storage. Legacy flat field.- TagFileID string `json:"tag_file_id,omitempty"`-- // EnableSummary enables summary extraction. Legacy flat field.- EnableSummary int `json:"enable_summary,omitempty"`-- // EnableMetadata enables automatic structured metadata extraction. Legacy flat field.- EnableMetadata int `json:"enable_metadata,omitempty"`-- // Metadata lists the target field definitions for- // EnableMetadata (mirrors parser_config.metadata / built_in_metadata:- // {key, type, description, enum}). When empty, EnableMetadata is a- // no-op (nothing to extract).- Metadata []common.MetadataFieldDef `json:"metadata,omitempty"`+ // Modular sub-configs+ Keywords KeywordExtractConfig `json:"keywords,omitempty"`+ Questions QuestionExtractConfig `json:"questions,omitempty"`+ Tags TagExtractConfig `json:"tags,omitempty"`+ Summary SummaryExtractConfig `json:"summary,omitempty"`+ Metadata MetadataExtractConfig `json:"metadata,omitempty"`}// Defaults returns the default ExtractorParam.func (ExtractorParam) Defaults() ExtractorParam {return ExtractorParam{- FieldName: "",- LLMID: "",- SystemPrompt: "",- Prompt: "",- AutoKeywords: 0,- AutoQuestions: 0,- AutoTags: 0,- TagFileID: "",- EnableSummary: 0,- EnableMetadata: 0,- Metadata: nil,+ LLMID: "",}}@@ -192,11 +141,9 @@ type ExtractorOutputs struct {// OutputFormat is always "chunks".OutputFormat string `json:"output_format,omitempty"`- // Chunks is the enriched chunk list. When the Extractor ran over- // a non-empty input list, each chunk gains a new key named after- // FieldName (e.g., field_name="summary" -> chunk["summary"]). When- // the Extractor ran over an empty input, Chunks contains a single- // entry with one key (FieldName) holding the LLM result.+ // Chunks is the enriched chunk list. Each chunk is enriched with+ // modular extraction fields (important_kwd, question_kwd, tag_kwd,+ // summary, metadata).Chunks []map[string]any `json:"chunks,omitempty"`// Error is set when the component short-circuits with an errordiff --git a/internal/ingestion/component/schema/schema_test.go b/internal/ingestion/component/schema/schema_test.goindex f3f7509e65..c27ae5976d 100644--- a/internal/ingestion/component/schema/schema_test.go+++ b/internal/ingestion/component/schema/schema_test.go@@ -504,8 +504,8 @@ func TestTokenizerOutputsJSONRoundTrip(t *testing.T) {func TestExtractorParamDefaults(t *testing.T) {p := ExtractorParam{}.Defaults()- if p.FieldName != "" {- t.Errorf("default field_name should be empty, got %q", p.FieldName)+ if p.LLMID != "" {+ t.Errorf("default llm_id should be empty, got %q", p.LLMID)}if err := p.Validate(); err != nil {t.Fatalf("default ExtractorParam should pass Validate, got %v", err)@@ -516,8 +516,8 @@ func TestExtractorParamValidate(t *testing.T) {if err := (&ExtractorParam{}).Validate(); err != nil {t.Fatalf("expected no error, got %v", err)}- if err := (&ExtractorParam{FieldName: "summary"}).Validate(); err != nil {- t.Fatalf("Validate with field_name should pass, got %v", err)+ if err := (&ExtractorParam{LLMID: "model@provider"}).Validate(); err != nil {+ t.Fatalf("Validate should pass, got %v", err)}}diff --git a/internal/ingestion/component/slice3_test.go b/internal/ingestion/component/slice3_test.godeleted file mode 100644index 4b64be01c4..0000000000--- a/internal/ingestion/component/slice3_test.go+++ /dev/null@@ -1,75 +0,0 @@-//-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.-//--// Slice 3 tests for port-rag-flow-pipeline-to-go.md Phase 5.-//-// Pins the Extractor prompt placeholder substitution.--package component--import (- "strings"- "testing"-)--// TestRenderExtractorPrompts_ReplacesAtChunks pins the-// `{ComponentName:ParamName@chunks}` placeholder substitution.-func TestRenderExtractorPrompts_ReplacesAtChunks(t *testing.T) {- prompt := "Extract metadata from: {TitleChunker:FlatMiceFix@chunks}"- ck := map[string]any{"text": "First chunk."}- _, got := renderExtractorPrompts("", prompt, ck, "First chunk.")- if strings.Contains(got, "{TitleChunker:FlatMiceFix@chunks}") {- t.Errorf("placeholder not substituted: %q", got)- }- if !strings.Contains(got, "First chunk.") {- t.Errorf("substitute missing chunk content: %q", got)- }- if n := strings.Count(got, "First chunk."); n != 1 {- t.Errorf("chunk text appears %d times, want 1: %q", n, got)- }-}--// TestRenderExtractorPrompts_LeavesUnknownPattern pins that unrecognized-// placeholders are preserved verbatim.-func TestRenderExtractorPrompts_LeavesUnknownPattern(t *testing.T) {- prompt := "Extract metadata from: {unknown_placeholder}"- _, got := renderExtractorPrompts("", prompt, nil, "")- if got != prompt {- t.Errorf("unknown placeholder: pattern should be preserved\n got: %q\n want: %q", got, prompt)- }-}--// TestRenderExtractorPrompts_NoPlaceholderAppendsChunkText pins the-// fallback append behavior when the prompt carries no body placeholder.-func TestRenderExtractorPrompts_NoPlaceholderAppendsChunkText(t *testing.T) {- prompt := "Plain prompt with no substitution."- ck := map[string]any{"text": "my chunk text"}- _, got := renderExtractorPrompts("", prompt, ck, "my chunk text")- want := "Plain prompt with no substitution.\n\nmy chunk text"- if got != want {- t.Errorf("got %q, want %q", got, want)- }-}--// TestRenderExtractorPrompts_SkipsEmptyChunkText pins that an empty-// chunkText does not append a trailing blank line.-func TestRenderExtractorPrompts_SkipsEmptyChunkText(t *testing.T) {- prompt := "Plain prompt"- _, got := renderExtractorPrompts("", prompt, map[string]any{}, "")- if got != prompt {- t.Errorf("empty chunkText should not alter prompt\n got: %q\n want: %q", got, prompt)- }-}diff --git a/internal/ingestion/pipeline/component_params.go b/internal/ingestion/pipeline/component_params.goindex fcb975ff91..f9c73e3e2b 100644--- a/internal/ingestion/pipeline/component_params.go+++ b/internal/ingestion/pipeline/component_params.go@@ -38,6 +38,16 @@ func ExtractAllComponentParams(dslJSON []byte) ([]ComponentParamsSchema, error)return nil, fmt.Errorf("ExtractAllComponentParams: DSL has no components")}+ extractorCount := 0+ for cpnID, comp := range dsl.Components {+ if isExtractorComponent(cpnID, comp.Obj.ComponentName) {+ extractorCount+++ }+ }+ if extractorCount > 1 {+ return nil, fmt.Errorf("ExtractAllComponentParams: at most 1 Extractor component is allowed, found %d", extractorCount)+ }+out := make([]ComponentParamsSchema, 0, len(dsl.Components))for cpnID, comp := range dsl.Components {params := make(map[string]any, len(comp.Obj.Params))diff --git a/internal/ingestion/pipeline/component_params_test.go b/internal/ingestion/pipeline/component_params_test.goindex ce8798f0a8..42244a2e14 100644--- a/internal/ingestion/pipeline/component_params_test.go+++ b/internal/ingestion/pipeline/component_params_test.go@@ -237,3 +237,34 @@ func TestExtractAllComponentParams_AllBuiltinTemplates(t *testing.T) {}t.Logf("tested %d builtin templates", tested)}++func TestExtractAllComponentParams_DisallowMultipleExtractors(t *testing.T) {+ dsl := map[string]any{+ "components": map[string]any{+ "Extractor:One": map[string]any{+ "obj": map[string]any{+ "component_name": "Extractor",+ "params": map[string]any{},+ },+ },+ "Extractor:Two": map[string]any{+ "obj": map[string]any{+ "component_name": "Extractor",+ "params": map[string]any{},+ },+ },+ },+ }+ dslJSON, err := json.Marshal(dsl)+ if err != nil {+ t.Fatalf("marshal dsl: %v", err)+ }++ _, err = ExtractAllComponentParams(dslJSON)+ if err == nil {+ t.Fatal("expected error for multiple Extractor components, got nil")+ }+ if !strings.Contains(err.Error(), "at most 1 Extractor") {+ t.Errorf("expected at most 1 Extractor error message, got: %v", err)+ }+}diff --git a/internal/ingestion/pipeline/pipeline.go b/internal/ingestion/pipeline/pipeline.goindex f43cf7ae50..8f50a92f69 100644--- a/internal/ingestion/pipeline/pipeline.go+++ b/internal/ingestion/pipeline/pipeline.go@@ -152,6 +152,9 @@ func NewPipelineFromDSL(dsl []byte, taskID string, opts ...PipelineOption) (*Pipif err != nil {return nil, fmt.Errorf("pipeline: decode canvas DSL: %w", err)}+ if err := ValidatePipeline(cnv); err != nil {+ return nil, err+ }// Capture the canonical canvas DSL bytes for the resume-time DSL// fingerprint. json.Marshal sorts map keys, so this is stable across// re-decodes of the same logical DSL (formatting-independent).@@ -170,6 +173,24 @@ func NewPipelineFromDSL(dsl []byte, taskID string, opts ...PipelineOption) (*Pipreturn p, nil}+// ValidatePipeline enforces ingestion pipeline constraints.+// Specifically, at most one Extractor component is permitted in the graph.+func ValidatePipeline(cnv *canvas.Canvas) error {+ if cnv == nil {+ return nil+ }+ extractorCount := 0+ for id, comp := range cnv.Components {+ if isExtractorComponent(id, comp.Obj.ComponentName) {+ extractorCount+++ }+ }+ if extractorCount > 1 {+ return fmt.Errorf("pipeline validation error: at most 1 Extractor component is allowed, found %d", extractorCount)+ }+ return nil+}+// WithComponentFactory installs an instance-scoped factory override for this// pipeline. It is used during canvas compilation so one pipeline run can// construct task-specific component instances without mutating the process-widediff --git a/internal/ingestion/pipeline/pipeline_params.go b/internal/ingestion/pipeline/pipeline_params.goindex d6784fe80c..82312e8cce 100644--- a/internal/ingestion/pipeline/pipeline_params.go+++ b/internal/ingestion/pipeline/pipeline_params.go@@ -2,19 +2,90 @@ package pipelineimport ("fmt"+ "reflect""strings""ragflow/internal/common""ragflow/internal/entity"+ "ragflow/internal/ingestion/component/schema""go.uber.org/zap")+// llmRuntimeParamKeys contains generic LLM hyper-parameters and UI switches common to LLM-based components.+var llmRuntimeParamKeys = map[string]struct{}{+ "llm_id": {},+ "temperature": {},+ "top_p": {},+ "max_tokens": {},+ "presence_penalty": {},+ "frequency_penalty": {},+ "outputs": {},+ "temperatureEnabled": {},+ "topPEnabled": {},+ "presencePenaltyEnabled": {},+ "frequencyPenaltyEnabled": {},+ "maxTokensEnabled": {},+}++// extractorValidParamKeys is dynamically derived from schema.ExtractorParam + LLM hyper-parameters.+var extractorValidParamKeys = func() map[string]struct{} {+ keys := extractJSONTags(schema.ExtractorParam{})+ for k := range llmRuntimeParamKeys {+ keys[k] = struct{}{}+ }+ return keys+}()++// extractJSONTags returns all top-level json tag names from a struct.+func extractJSONTags(v any) map[string]struct{} {+ tags := make(map[string]struct{})+ t := reflect.TypeOf(v)+ if t == nil {+ return tags+ }+ if t.Kind() == reflect.Ptr {+ t = t.Elem()+ }+ if t.Kind() != reflect.Struct {+ return tags+ }+ for i := 0; i < t.NumField(); i++ {+ tag := t.Field(i).Tag.Get("json")+ if tag == "" || tag == "-" {+ continue+ }+ name := strings.Split(tag, ",")[0]+ if name != "" {+ tags[name] = struct{}{}+ }+ }+ return tags+}++// isExtractorComponent returns true if the component name or component ID+// indicates an Extractor component.+func isExtractorComponent(cpnID, componentName string) bool {+ lowerName := strings.ToLower(componentName)+ lowerID := strings.ToLower(cpnID)+ return lowerName == "extractor" ||+ strings.HasPrefix(lowerID, "extractor:") ||+ strings.HasPrefix(lowerID, "extractor_")+}++// getComponentParamWhitelist returns the dynamic parameter whitelist for a component type.+func getComponentParamWhitelist(cpnID string) (map[string]struct{}, bool) {+ if isExtractorComponent(cpnID, "") {+ return extractorValidParamKeys, true+ }+ return nil, false+}+// CleanComponentParams filters rawConfig against the DSL schema given by dslJSON.// Keys containing ':' are treated as component IDs; they are kept only when both-// the cpnID AND the param name exist in the DSL schema. Keys without ':' (legacy-// flat fields such as chunk_token_num, image_context_size) are dropped with a-// warning — they do not belong in the new component-params world.+// the cpnID AND the param name exist in the DSL schema or the component's dynamic+// parameter schema (e.g. Extractor modular features). Keys without ':' (legacy+// flat fields) are dropped with a warning.func CleanComponentParams(dslJSON []byte, rawConfig map[string]interface{}) map[string]interface{} {schemas, err := ExtractAllComponentParams(dslJSON)if err != nil {@@ -49,18 +120,22 @@ func CleanComponentParams(dslJSON []byte, rawConfig map[string]interface{}) map[if !ok {continue}+ dynamicWhitelist, hasDynamic := getComponentParamWhitelist(key)+ if hasDynamic {+ params = NormalizeExtractorParams(params)+ }cleaned := make(map[string]any, len(params))for pk, pv := range params {if _, ok := validKeys[pk]; ok {cleaned[pk] = pvcontinue}- // Extractor params are valid even when a builtin template does- // not declare them explicitly in params: {} (the frontend stores- // them on the node params); keep them through the filter.- if isExtractorComponent(key) && isExtractorParam(pk) {- cleaned[pk] = pv- continue+ // Extractor and dynamic components: check reflection-driven whitelist+ if hasDynamic {+ if _, ok := dynamicWhitelist[pk]; ok {+ cleaned[pk] = pv+ continue+ }}common.Warn("CleanComponentParams: dropping unknown param",zap.String("cpnID", key), zap.String("param", pk))@@ -72,6 +147,64 @@ func CleanComponentParams(dslJSON []byte, rawConfig map[string]interface{}) map[return result}+// NormalizeExtractorParams normalizes a raw Extractor parameters map into the canonical modular format.+func NormalizeExtractorParams(raw map[string]any) map[string]any {+ if raw == nil {+ return map[string]any{}+ }+ out := make(map[string]any, len(raw))+ for k, v := range raw {+ out[k] = v+ }++ // 1. Keywords+ if kwRaw, ok := out["keywords"].(map[string]any); ok {+ kw := make(map[string]any, len(kwRaw))+ for k, v := range kwRaw {+ kw[k] = v+ }+ out["keywords"] = kw+ }++ // 2. Questions+ if qRaw, ok := out["questions"].(map[string]any); ok {+ q := make(map[string]any, len(qRaw))+ for k, v := range qRaw {+ q[k] = v+ }+ out["questions"] = q+ }++ // 3. Tags+ if tagRaw, ok := out["tags"].(map[string]any); ok {+ tag := make(map[string]any, len(tagRaw))+ for k, v := range tagRaw {+ tag[k] = v+ }+ out["tags"] = tag+ }++ // 4. Summary+ if sumRaw, ok := out["summary"].(map[string]any); ok {+ sum := make(map[string]any, len(sumRaw))+ for k, v := range sumRaw {+ sum[k] = v+ }+ out["summary"] = sum+ }++ // 5. Metadata+ if metaRaw, ok := out["metadata"].(map[string]any); ok {+ meta := make(map[string]any, len(metaRaw))+ for k, v := range metaRaw {+ meta[k] = v+ }+ out["metadata"] = meta+ }++ return out+}+// BuildParserConfig builds the final parser_config by starting from the DSL// defaults for every component, then overlaying the cleaned incoming overrides.// This ensures all components from the current pipeline are present while@@ -156,28 +289,3 @@ func ResolveComponentParamsDefaultsFromIDs(parserID string, pipelineID *string)}return out, nil}--// isExtractorComponent reports whether a component id is an Extractor node-// (component ids are "<ComponentName>:<name>" or "<ComponentName>_<name>").-func isExtractorComponent(cpnID string) bool {- lower := strings.ToLower(cpnID)- return strings.HasPrefix(lower, "extractor:") || strings.HasPrefix(lower, "extractor_")-}--// isExtractorParam reports whether a param belongs to the Extractor component-// (modular sub-configs, legacy flat fields, LLM settings, or UI state flags).-func isExtractorParam(param string) bool {- switch param {- case "keywords", "questions", "tags", "summary", "metadata_config",- "auto_keywords", "keywords_sys_prompt",- "auto_questions", "questions_sys_prompt",- "auto_tags", "tag_file_id",- "enable_summary", "sys_prompt", "system_prompt", "prompt", "prompts", "field_name",- "enable_metadata", "metadata", "built_in_metadata",- "llm_id", "temperature", "top_p", "max_tokens", "presence_penalty", "frequency_penalty",- "temperatureEnabled", "topPEnabled", "maxTokensEnabled", "presencePenaltyEnabled", "frequencyPenaltyEnabled",- "outputs":- return true- }- return false-}diff --git a/internal/ingestion/pipeline/pipeline_params_test.go b/internal/ingestion/pipeline/pipeline_params_test.goindex 1f544dc05d..fbe0aee400 100644--- a/internal/ingestion/pipeline/pipeline_params_test.go+++ b/internal/ingestion/pipeline/pipeline_params_test.go@@ -239,15 +239,21 @@ func TestBuildParserConfig_BuiltinExtractorKeepsTagFileID(t *testing.T) {}checked++overrides := map[string]any{- s.CpnID: map[string]any{"tag_file_id": "file-123"},+ s.CpnID: map[string]any{+ "tags": map[string]any{"top_n": 5, "tag_file_id": "file-123"},+ },}result := BuildParserConfig(dslJSON, overrides)params, ok := result[s.CpnID].(map[string]any)if !ok {t.Fatalf("template %q: expected component %q in result", ref, s.CpnID)}- if params["tag_file_id"] != "file-123" {- t.Errorf("template %q: expected tag_file_id to survive BuildParserConfig, got %v", ref, params["tag_file_id"])+ var tagFileID any+ if tags, ok := params["tags"].(map[string]any); ok {+ tagFileID = tags["tag_file_id"]+ }+ if tagFileID != "file-123" {+ t.Errorf("template %q: expected tag_file_id to survive BuildParserConfig, got %v", ref, tagFileID)}}}@@ -335,8 +341,11 @@ func TestBuildParserConfig_BuiltinExtractorKeepsBuiltInMetadata(t *testing.T) {checked++overrides := map[string]any{s.CpnID: map[string]any{- "built_in_metadata": []any{- map[string]any{"key": "update_time", "type": "time"},+ "metadata": map[string]any{+ "enabled": true,+ "built_in_metadata": []any{+ map[string]any{"key": "update_time", "type": "time"},+ },},},}@@ -345,8 +354,8 @@ func TestBuildParserConfig_BuiltinExtractorKeepsBuiltInMetadata(t *testing.T) {if !ok {t.Fatalf("template %q: expected component %q in result", ref, s.CpnID)}- if _, ok := params["built_in_metadata"]; !ok {- t.Errorf("template %q: built_in_metadata dropped by CleanComponentParams; add \"built_in_metadata\": [] to the extractor node params", ref)+ if _, ok := params["metadata"]; !ok {+ t.Errorf("template %q: metadata dropped by CleanComponentParams", ref)}}}@@ -378,7 +387,7 @@ func TestCleanComponentParams_KeepsModularExtractorParams(t *testing.T) {"enabled": true,"system_prompt": "summary prompt",},- "metadata_config": map[string]any{+ "metadata": map[string]any{"enabled": true,"metadata": []any{map[string]any{"key": "category", "type": "string"},@@ -387,9 +396,9 @@ func TestCleanComponentParams_KeepsModularExtractorParams(t *testing.T) {"keywords": map[string]any{"top_n": 5,},- "enable_summary": 1,"llm_id": "gpt-4","temperature": 0.7,+ "enable_summary": 1, // legacy flat param should be dropped},}@@ -402,8 +411,8 @@ func TestCleanComponentParams_KeepsModularExtractorParams(t *testing.T) {if sum, ok := ext["summary"].(map[string]any); !ok || sum["enabled"] != true {t.Errorf("expected summary.enabled == true, got: %v", ext["summary"])}- if meta, ok := ext["metadata_config"].(map[string]any); !ok || meta["enabled"] != true {- t.Errorf("expected metadata_config.enabled == true, got: %v", ext["metadata_config"])+ if meta, ok := ext["metadata"].(map[string]any); !ok || meta["enabled"] != true {+ t.Errorf("expected metadata.enabled == true, got: %v", ext["metadata"])}if kw, ok := ext["keywords"].(map[string]any); !ok || kw["top_n"] != 5 {t.Errorf("expected keywords.top_n == 5, got: %v", ext["keywords"])@@ -411,4 +420,172 @@ func TestCleanComponentParams_KeepsModularExtractorParams(t *testing.T) {if ext["llm_id"] != "gpt-4" {t.Errorf("expected llm_id == gpt-4, got: %v", ext["llm_id"])}+ if _, ok := ext["enable_summary"]; ok {+ t.Errorf("expected legacy flat field enable_summary to be dropped, got: %v", ext["enable_summary"])+ }+}++func TestCleanComponentParams_ExtractorBuiltInMetadataRetained(t *testing.T) {+ dslJSON := []byte(`{+ "components": {+ "Extractor:AutoExtractDefault": {+ "obj": {+ "component_name": "Extractor",+ "params": {+ "outputs": {},+ "metadata": {}+ }+ }+ }+ }+ }`)++ rawConfig := map[string]any{+ "Extractor:AutoExtractDefault": map[string]any{+ "metadata": map[string]any{+ "enabled": true,+ "metadata": []map[string]any{+ {"key": "custom_field", "type": "string"},+ },+ "built_in_metadata": []map[string]any{+ {"key": "doc_name", "type": "string"},+ },+ },+ },+ }++ result := CleanComponentParams(dslJSON, rawConfig)+ ext, ok := result["Extractor:AutoExtractDefault"].(map[string]any)+ if !ok {+ t.Fatalf("expected Extractor:AutoExtractDefault in result, got: %v", result)+ }++ meta, ok := ext["metadata"].(map[string]any)+ if !ok {+ t.Fatalf("expected metadata object, got: %#v", ext["metadata"])+ }+ if meta["enabled"] != true {+ t.Errorf("expected enabled == true, got: %#v", meta["enabled"])+ }+ mFields, ok := meta["metadata"].([]map[string]any)+ if !ok || len(mFields) != 1 {+ t.Errorf("expected 1 metadata field, got: %#v", meta["metadata"])+ }+ bFields, ok := meta["built_in_metadata"].([]map[string]any)+ if !ok || len(bFields) != 1 {+ t.Errorf("expected 1 built_in_metadata field, got: %#v", meta["built_in_metadata"])+ }+}++func TestNormalizeExtractorParams_TableDriven(t *testing.T) {+ tests := []struct {+ name string+ input map[string]any+ validate func(t *testing.T, out map[string]any)+ }{+ {+ name: "nil input returns empty map",+ input: nil,+ validate: func(t *testing.T, out map[string]any) {+ if len(out) != 0 {+ t.Errorf("expected empty map for nil input, got %#v", out)+ }+ },+ },+ {+ name: "modular keywords preserved",+ input: map[string]any{+ "keywords": map[string]any{+ "top_n": 5,+ "system_prompt": "custom kw prompt",+ },+ },+ validate: func(t *testing.T, out map[string]any) {+ kw, ok := out["keywords"].(map[string]any)+ if !ok || kw["top_n"] != 5 || kw["system_prompt"] != "custom kw prompt" {+ t.Errorf("keywords mismatch: %#v", out["keywords"])+ }+ },+ },+ {+ name: "modular questions preserved",+ input: map[string]any{+ "questions": map[string]any{+ "top_n": 3,+ "system_prompt": "custom q prompt",+ },+ },+ validate: func(t *testing.T, out map[string]any) {+ q, ok := out["questions"].(map[string]any)+ if !ok || q["top_n"] != 3 || q["system_prompt"] != "custom q prompt" {+ t.Errorf("questions mismatch: %#v", out["questions"])+ }+ },+ },+ {+ name: "modular tags preserved",+ input: map[string]any{+ "tags": map[string]any{+ "top_n": 7,+ "tag_file_id": "file-123",+ },+ },+ validate: func(t *testing.T, out map[string]any) {+ tag, ok := out["tags"].(map[string]any)+ if !ok || tag["top_n"] != 7 || tag["tag_file_id"] != "file-123" {+ t.Errorf("tags mismatch: %#v", out["tags"])+ }+ },+ },+ {+ name: "modular summary preserved",+ input: map[string]any{+ "summary": map[string]any{+ "enabled": true,+ "system_prompt": "summary prompt",+ },+ },+ validate: func(t *testing.T, out map[string]any) {+ sum, ok := out["summary"].(map[string]any)+ if !ok || sum["enabled"] != true || sum["system_prompt"] != "summary prompt" {+ t.Errorf("summary mismatch: %#v", out["summary"])+ }+ },+ },+ {+ name: "modular metadata preserved",+ input: map[string]any{+ "metadata": map[string]any{+ "enabled": true,+ "metadata": []map[string]any{+ {"key": "cat", "type": "string"},+ },+ "built_in_metadata": []map[string]any{+ {"key": "file_name", "type": "string"},+ },+ },+ },+ validate: func(t *testing.T, out map[string]any) {+ meta, ok := out["metadata"].(map[string]any)+ if !ok || meta["enabled"] != true {+ t.Fatalf("expected enabled == true, got %#v", out["metadata"])+ }+ mList, ok := meta["metadata"].([]map[string]any)+ if !ok || len(mList) != 1 {+ t.Errorf("metadata slice mismatch: %#v", meta["metadata"])+ }+ bList, ok := meta["built_in_metadata"].([]map[string]any)+ if !ok || len(bList) != 1 {+ t.Errorf("built_in_metadata slice mismatch: %#v", meta["built_in_metadata"])+ }+ },+ },+ }++ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ out := NormalizeExtractorParams(tc.input)+ tc.validate(t, out)+ })+ }}diff --git a/internal/ingestion/pipeline/pipeline_test.go b/internal/ingestion/pipeline/pipeline_test.goindex 1aa9970b22..79b1ad4731 100644--- a/internal/ingestion/pipeline/pipeline_test.go+++ b/internal/ingestion/pipeline/pipeline_test.go@@ -20,6 +20,7 @@ import ("context""errors""fmt"+ "strings""sync""testing""time"@@ -909,3 +910,25 @@ func TestRunPlain_WithTracker_Error(t *testing.T) {t.Fatal("expected stage error, got nil")}}++func TestValidatePipeline_DisallowMultipleExtractors(t *testing.T) {+ dsl := []byte(`{+ "dsl": {+ "components": {+ "begin": {"obj": {"component_name": "Begin", "params": {}}, "downstream": ["Extractor:One"]},+ "Extractor:One": {"obj": {"component_name": "Extractor", "params": {}}, "upstream": ["begin"], "downstream": ["Extractor:Two"]},+ "Extractor:Two": {"obj": {"component_name": "Extractor", "params": {}}, "upstream": ["Extractor:One"]}+ },+ "path": ["begin", "Extractor:One", "Extractor:Two"],+ "graph": {"nodes": []}+ }+ }`)++ _, err := NewPipelineFromDSL(dsl, "test-task-dup-ext")+ if err == nil {+ t.Fatal("expected validation error for multiple Extractor components, got nil")+ }+ if !strings.Contains(err.Error(), "at most 1 Extractor component is allowed") {+ t.Errorf("expected at most 1 Extractor error message, got: %v", err)+ }+}diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_audio.json b/internal/ingestion/pipeline/template/ingestion_pipeline_audio.jsonindex 90b6e82366..924cb74476 100644--- a/internal/ingestion/pipeline/template/ingestion_pipeline_audio.json+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_audio.json@@ -125,20 +125,25 @@"obj": {"component_name": "Extractor","params": {- "field_name": "",- "outputs": {- "chunks": {- "type": "Array<Object>",- "value": []- }- },- "auto_keywords": 0,- "auto_questions": 0,"llm_id": "",- "auto_tags": 0,- "tag_file_id": "",- "enable_metadata": 0,- "metadata": []+ "keywords": {+ "top_n": 0+ },+ "questions": {+ "top_n": 0+ },+ "tags": {+ "top_n": 0,+ "tag_file_id": ""+ },+ "summary": {+ "enabled": false+ },+ "metadata": {+ "enabled": false,+ "metadata": [],+ "built_in_metadata": []+ }}}}diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_book.json b/internal/ingestion/pipeline/template/ingestion_pipeline_book.jsonindex ddc987eabe..09fa90bdbb 100644--- a/internal/ingestion/pipeline/template/ingestion_pipeline_book.json+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_book.json@@ -186,20 +186,25 @@"obj": {"component_name": "Extractor","params": {- "field_name": "",- "outputs": {- "chunks": {- "type": "Array<Object>",- "value": []- }- },- "auto_keywords": 0,- "auto_questions": 0,"llm_id": "",- "auto_tags": 0,- "tag_file_id": "",- "enable_metadata": 0,- "metadata": []+ "keywords": {+ "top_n": 0+ },+ "questions": {+ "top_n": 0+ },+ "tags": {+ "top_n": 0,+ "tag_file_id": ""+ },+ "summary": {+ "enabled": false+ },+ "metadata": {+ "enabled": false,+ "metadata": [],+ "built_in_metadata": []+ }}}}diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_email.json b/internal/ingestion/pipeline/template/ingestion_pipeline_email.jsonindex 09c58c291e..a901ef0795 100644--- a/internal/ingestion/pipeline/template/ingestion_pipeline_email.json+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_email.json@@ -130,20 +130,25 @@"obj": {"component_name": "Extractor","params": {- "field_name": "",- "outputs": {- "chunks": {- "type": "Array<Object>",- "value": []- }- },- "auto_keywords": 0,- "auto_questions": 0,"llm_id": "",- "auto_tags": 0,- "tag_file_id": "",- "enable_metadata": 0,- "metadata": []+ "keywords": {+ "top_n": 0+ },+ "questions": {+ "top_n": 0+ },+ "tags": {+ "top_n": 0,+ "tag_file_id": ""+ },+ "summary": {+ "enabled": false+ },+ "metadata": {+ "enabled": false,+ "metadata": [],+ "built_in_metadata": []+ }}}}diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_general.json b/internal/ingestion/pipeline/template/ingestion_pipeline_general.jsonindex 22d0d6fe7e..3374a6696f 100644--- a/internal/ingestion/pipeline/template/ingestion_pipeline_general.json+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_general.json@@ -214,20 +214,25 @@"obj": {"component_name": "Extractor","params": {- "field_name": "",- "outputs": {- "chunks": {- "type": "Array<Object>",- "value": []- }- },- "auto_keywords": 0,- "auto_questions": 0,"llm_id": "",- "auto_tags": 0,- "tag_file_id": "",- "enable_metadata": 0,- "metadata": []+ "keywords": {+ "top_n": 0+ },+ "questions": {+ "top_n": 0+ },+ "tags": {+ "top_n": 0,+ "tag_file_id": ""+ },+ "summary": {+ "enabled": false+ },+ "metadata": {+ "enabled": false,+ "metadata": [],+ "built_in_metadata": []+ }}}}diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_laws.json b/internal/ingestion/pipeline/template/ingestion_pipeline_laws.jsonindex b08be5d874..b1b80c2acd 100644--- a/internal/ingestion/pipeline/template/ingestion_pipeline_laws.json+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_laws.json@@ -199,20 +199,25 @@"obj": {"component_name": "Extractor","params": {- "field_name": "",- "outputs": {- "chunks": {- "type": "Array<Object>",- "value": []- }- },- "auto_keywords": 0,- "auto_questions": 0,"llm_id": "",- "auto_tags": 0,- "tag_file_id": "",- "enable_metadata": 0,- "metadata": []+ "keywords": {+ "top_n": 0+ },+ "questions": {+ "top_n": 0+ },+ "tags": {+ "top_n": 0,+ "tag_file_id": ""+ },+ "summary": {+ "enabled": false+ },+ "metadata": {+ "enabled": false,+ "metadata": [],+ "built_in_metadata": []+ }}}}diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json b/internal/ingestion/pipeline/template/ingestion_pipeline_manual.jsonindex 1faad567e9..2809b48305 100644--- a/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json@@ -166,20 +166,25 @@"obj": {"component_name": "Extractor","params": {- "field_name": "",- "outputs": {- "chunks": {- "type": "Array<Object>",- "value": []- }- },- "auto_keywords": 0,- "auto_questions": 0,"llm_id": "",- "auto_tags": 0,- "tag_file_id": "",- "enable_metadata": 0,- "metadata": []+ "keywords": {+ "top_n": 0+ },+ "questions": {+ "top_n": 0+ },+ "tags": {+ "top_n": 0,+ "tag_file_id": ""+ },+ "summary": {+ "enabled": false+ },+ "metadata": {+ "enabled": false,+ "metadata": [],+ "built_in_metadata": []+ }}}}diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_one.json b/internal/ingestion/pipeline/template/ingestion_pipeline_one.jsonindex 3406161af5..0bcfb58a44 100644--- a/internal/ingestion/pipeline/template/ingestion_pipeline_one.json+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_one.json@@ -179,20 +179,25 @@"obj": {"component_name": "Extractor","params": {- "field_name": "",- "outputs": {- "chunks": {- "type": "Array<Object>",- "value": []- }- },- "auto_keywords": 0,- "auto_questions": 0,"llm_id": "",- "auto_tags": 0,- "tag_file_id": "",- "enable_metadata": 0,- "metadata": []+ "keywords": {+ "top_n": 0+ },+ "questions": {+ "top_n": 0+ },+ "tags": {+ "top_n": 0,+ "tag_file_id": ""+ },+ "summary": {+ "enabled": false+ },+ "metadata": {+ "enabled": false,+ "metadata": [],+ "built_in_metadata": []+ }}}}diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_paper.json b/internal/ingestion/pipeline/template/ingestion_pipeline_paper.jsonindex 07a49d26e2..a679e801b6 100644--- a/internal/ingestion/pipeline/template/ingestion_pipeline_paper.json+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_paper.json@@ -147,20 +147,25 @@"obj": {"component_name": "Extractor","params": {- "field_name": "",- "outputs": {- "chunks": {- "type": "Array<Object>",- "value": []- }- },- "auto_keywords": 0,- "auto_questions": 0,"llm_id": "",- "auto_tags": 0,- "tag_file_id": "",- "enable_metadata": 0,- "metadata": []+ "keywords": {+ "top_n": 0+ },+ "questions": {+ "top_n": 0+ },+ "tags": {+ "top_n": 0,+ "tag_file_id": ""+ },+ "summary": {+ "enabled": false+ },+ "metadata": {+ "enabled": false,+ "metadata": [],+ "built_in_metadata": []+ }}}}diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_picture.json b/internal/ingestion/pipeline/template/ingestion_pipeline_picture.jsonindex effb4e5b7d..5d6fe299bd 100644--- a/internal/ingestion/pipeline/template/ingestion_pipeline_picture.json+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_picture.json@@ -139,20 +139,25 @@"obj": {"component_name": "Extractor","params": {- "field_name": "",- "outputs": {- "chunks": {- "type": "Array<Object>",- "value": []- }- },- "auto_keywords": 0,- "auto_questions": 0,"llm_id": "",- "auto_tags": 0,- "tag_file_id": "",- "enable_metadata": 0,- "metadata": []+ "keywords": {+ "top_n": 0+ },+ "questions": {+ "top_n": 0+ },+ "tags": {+ "top_n": 0,+ "tag_file_id": ""+ },+ "summary": {+ "enabled": false+ },+ "metadata": {+ "enabled": false,+ "metadata": [],+ "built_in_metadata": []+ }}}}diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_presentation.json b/internal/ingestion/pipeline/template/ingestion_pipeline_presentation.jsonindex b4746fff1c..c10eab6733 100644--- a/internal/ingestion/pipeline/template/ingestion_pipeline_presentation.json+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_presentation.json@@ -125,20 +125,25 @@"obj": {"component_name": "Extractor","params": {- "field_name": "",- "outputs": {- "chunks": {- "type": "Array<Object>",- "value": []- }- },- "auto_keywords": 0,- "auto_questions": 0,"llm_id": "",- "auto_tags": 0,- "tag_file_id": "",- "enable_metadata": 0,- "metadata": []+ "keywords": {+ "top_n": 0+ },+ "questions": {+ "top_n": 0+ },+ "tags": {+ "top_n": 0,+ "tag_file_id": ""+ },+ "summary": {+ "enabled": false+ },+ "metadata": {+ "enabled": false,+ "metadata": [],+ "built_in_metadata": []+ }}}}diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.goindex d63fba4611..3d224c8121 100644--- a/internal/ingestion/task/pipeline_executor.go+++ b/internal/ingestion/task/pipeline_executor.go@@ -309,7 +309,8 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput mapfunc builtInMetadataFromParserConfig(parserConfig entity.JSONMap) ([]any, bool) {var extractorKeys []stringfor k := range parserConfig {- if strings.HasPrefix(k, "Extractor") {+ lower := strings.ToLower(k)+ if strings.HasPrefix(lower, "extractor:") || strings.HasPrefix(lower, "extractor_") {extractorKeys = append(extractorKeys, k)}}@@ -318,6 +319,12 @@ func builtInMetadataFromParserConfig(parserConfig entity.JSONMap) ([]any, bool)for _, k := range extractorKeys {nodeRaw := parserConfig[k]if node, ok := nodeRaw.(map[string]any); ok {+ if metaObj, ok := node["metadata"].(map[string]any); ok {+ enabled := parserConfigBool(metaObj["enabled"])+ if arr, ok := metaObj["built_in_metadata"].([]any); ok && len(arr) > 0 {+ return arr, enabled+ }+ }if metaConf, ok := node["metadata_config"].(map[string]any); ok {enabled := parserConfigBool(metaConf["enabled"])if arr, ok := metaConf["built_in_metadata"].([]any); ok && len(arr) > 0 {@@ -330,14 +337,18 @@ func builtInMetadataFromParserConfig(parserConfig entity.JSONMap) ([]any, bool)}}}+ if metaObj, ok := parserConfig["metadata"].(map[string]any); ok {+ if arr, ok := metaObj["built_in_metadata"].([]any); ok && len(arr) > 0 {+ return arr, true+ }+ }if metaConf, ok := parserConfig["metadata_config"].(map[string]any); ok {- enabled := parserConfigBool(metaConf["enabled"])if arr, ok := metaConf["built_in_metadata"].([]any); ok && len(arr) > 0 {- return arr, enabled+ return arr, true}}if arr, ok := parserConfig["built_in_metadata"].([]any); ok && len(arr) > 0 {- return arr, parserConfigBool(parserConfig["enable_metadata"])+ return arr, true}return nil, false}diff --git a/internal/ingestion/task/pipeline_executor_defaults_test.go b/internal/ingestion/task/pipeline_executor_defaults_test.goindex 6d9fcca7be..0330babc95 100644--- a/internal/ingestion/task/pipeline_executor_defaults_test.go+++ b/internal/ingestion/task/pipeline_executor_defaults_test.go@@ -38,16 +38,16 @@ import (// Comparison is done per-component (see assertComponentsMatch), so the JSON// serialization order inside each entry is irrelevant.var builtinComponentParamsGolden = map[string]string{- "audio": "{\"File\": {}, \"Parser:SongsFillAir\": {\"audio\": {\"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"aac\", \"aiff\", \"ape\", \"au\", \"da\", \"flac\", \"midi\", \"mp3\", \"ogg\", \"oggvorbis\", \"realaudio\", \"vqf\", \"wav\", \"wave\", \"wma\"]}}, \"TokenChunker:BlueSkiesLaugh\": {}, \"Tokenizer:KindEyesWatch\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",- "book": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"remove_toc\": true, \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:GrumpyGarlicsBake\": {\"hierarchy\": 5, \"include_heading_content\": true, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"hierarchy\"}, \"Tokenizer:HotDonutsRing\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",- "email": "{\"File\": {}, \"Parser:BirdsFlutterHigh\": {\"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\"]}}, \"TokenChunker:WarmBreadSmells\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:NiceWordsSpoken\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",- "general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",- "laws": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"remove_toc\": true, \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:SpicyKeysKick\": {\"hierarchy\": 2, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"hierarchy\"}, \"Tokenizer:PublicJobsTake\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",- "manual": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"ManualChunker:NineInsectsFind\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:FunnyBalloonsGrin\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",- "one": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"OneChunker:DryDrinksVisit\": {}, \"Tokenizer:FrankWeeksListen\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",- "paper": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"pdf\": {\"enable_multi_column\": true, \"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"TitleChunker:SparklySchoolsTravel\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:GreatCarsWash\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",- "picture": "{\"File\": {}, \"Parser:ViewsCaptureLight\": {\"image\": {\"output_format\": \"json\", \"parse_method\": \"ocr\", \"preprocess\": [\"main_content\"], \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"video\": {\"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"3gp\", \"3gpp\", \"avi\", \"flv\", \"mkv\", \"mov\", \"mp4\", \"mpeg\", \"mpg\", \"webm\", \"wmv\"]}}, \"TokenChunker:BrightColorsGlow\": {}, \"Tokenizer:SharpLensFocus\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",- "presentation": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"slides\": {\"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pptx\", \"ppt\"]}}, \"Tokenizer:TallTreesDance\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"PresentationChunker:HappyHillsGlow\": {}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",+ "audio": "{\"File\": {}, \"Parser:SongsFillAir\": {\"audio\": {\"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"aac\", \"aiff\", \"ape\", \"au\", \"da\", \"flac\", \"midi\", \"mp3\", \"ogg\", \"oggvorbis\", \"realaudio\", \"vqf\", \"wav\", \"wave\", \"wma\"]}}, \"TokenChunker:BlueSkiesLaugh\": {}, \"Tokenizer:KindEyesWatch\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",+ "book": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"remove_toc\": true, \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:GrumpyGarlicsBake\": {\"hierarchy\": 5, \"include_heading_content\": true, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"hierarchy\"}, \"Tokenizer:HotDonutsRing\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",+ "email": "{\"File\": {}, \"Parser:BirdsFlutterHigh\": {\"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\"]}}, \"TokenChunker:WarmBreadSmells\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:NiceWordsSpoken\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",+ "general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",+ "laws": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"remove_toc\": true, \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:SpicyKeysKick\": {\"hierarchy\": 2, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"hierarchy\"}, \"Tokenizer:PublicJobsTake\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",+ "manual": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"ManualChunker:NineInsectsFind\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:FunnyBalloonsGrin\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",+ "one": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"OneChunker:DryDrinksVisit\": {}, \"Tokenizer:FrankWeeksListen\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",+ "paper": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"pdf\": {\"enable_multi_column\": true, \"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"TitleChunker:SparklySchoolsTravel\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:GreatCarsWash\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",+ "picture": "{\"File\": {}, \"Parser:ViewsCaptureLight\": {\"image\": {\"output_format\": \"json\", \"parse_method\": \"ocr\", \"preprocess\": [\"main_content\"], \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"video\": {\"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"3gp\", \"3gpp\", \"avi\", \"flv\", \"mkv\", \"mov\", \"mp4\", \"mpeg\", \"mpg\", \"webm\", \"wmv\"]}}, \"TokenChunker:BrightColorsGlow\": {}, \"Tokenizer:SharpLensFocus\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",+ "presentation": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"slides\": {\"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pptx\", \"ppt\"]}}, \"Tokenizer:TallTreesDance\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"PresentationChunker:HappyHillsGlow\": {}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}","qa": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"Tokenizer:ColdCloudsDream\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"QAChunker:TidyCloudsThink\": {}}","table": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}, \"column_mode\": \"auto\", \"column_roles\": {}, \"column_names\": []}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TableChunker:FastFoxesJump\": {}, \"Tokenizer:DeepLakesShine\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}}","knowledge_compiler": "{\"File\": {}, \"Compiler:KnownSwiftLions\": {\"compilation_template_group_id\": \"\", \"llm_id\": \"\", \"plan\": false}, \"Parser:HipSignsRhyme\": {\"setups\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}}",diff --git a/internal/ingestion/task/pipeline_executor_test.go b/internal/ingestion/task/pipeline_executor_test.goindex 80222d12f6..62a50f0bc4 100644--- a/internal/ingestion/task/pipeline_executor_test.go+++ b/internal/ingestion/task/pipeline_executor_test.go@@ -575,3 +575,25 @@ func TestCountOriginalChunkIDs(t *testing.T) {t.Fatalf("compiler products: got %d, want 2", n)}}++func TestBuiltInMetadata_ZeroExtractorFallback(t *testing.T) {+ // Scenario: User constructed a pipeline with 0 Extractor components,+ // but dataset parser_config has top-level built_in_metadata.+ parserConfig := map[string]any{+ "Parser:HipSignsRhyme": map[string]any{+ "setups": map[string]any{},+ },+ "built_in_metadata": []any{+ map[string]any{"key": "file_name", "type": "string"},+ map[string]any{"key": "update_time", "type": "time"},+ },+ }++ arr, enabled := builtInMetadataFromParserConfig(parserConfig)+ if !enabled {+ t.Errorf("expected builtInMetadataFromParserConfig enabled=true for zero-extractor pipeline with top-level built_in_metadata")+ }+ if len(arr) != 2 {+ t.Errorf("expected 2 built-in metadata items, got %d", len(arr))+ }+}diff --git a/internal/service/component_scoped_parser_config.go b/internal/service/component_scoped_parser_config.goindex 55ca470129..cf78bc993f 100644--- a/internal/service/component_scoped_parser_config.go+++ b/internal/service/component_scoped_parser_config.go@@ -18,8 +18,17 @@ func ApplyComponentScopedParserConfig(}enableMetadata := parserConfigTruthy(parserConfig["enable_metadata"])- metadataFields := mergeMetadataFields(parserConfig)- hasMetadataConfig := hasMetadataConfigShape(parserConfig)+ if !enableMetadata {+ if metaMap, ok := parserConfig["metadata"].(map[string]any); ok {+ enableMetadata = parserConfigTruthy(metaMap["enabled"])+ }+ }+ if !enableMetadata {+ enableMetadata = parserConfigTruthy(parserConfig["enabled"])+ }++ metadataFields := extractFieldList(parserConfig, "fields", "metadata")+ builtInMetadataFields := extractFieldList(parserConfig, "built_in_metadata", "")for cpnID, raw := range parserConfig {params, ok := raw.(map[string]any)@@ -33,13 +42,46 @@ func ApplyComponentScopedParserConfig(if value, _ := params["llm_id"].(string); strings.TrimSpace(value) == "" && strings.TrimSpace(llmID) != "" {params["llm_id"] = llmID}- if enableMetadata && len(metadataFields) > 0 {- params["enable_metadata"] = 1- params["metadata"] = metadataFields- } else if hasMetadataConfig {- params["enable_metadata"] = 0- params["metadata"] = []any{}+ delete(params, "enable_metadata")++ hasTopLevelFields := len(metadataFields) > 0 || len(builtInMetadataFields) > 0++ if hasTopLevelFields {+ params["metadata"] = map[string]any{+ "enabled": enableMetadata,+ "metadata": metadataFields,+ "built_in_metadata": builtInMetadataFields,+ }+ } else {+ nodeCustom := extractFieldList(params, "fields", "metadata")+ nodeBuiltIn := extractFieldList(params, "built_in_metadata", "")+ nodeEnabled := false++ if metaMap, ok := params["metadata"].(map[string]any); ok && metaMap["enabled"] != nil {+ nodeEnabled = parserConfigTruthy(metaMap["enabled"])+ } else if mcMap, ok := params["metadata_config"].(map[string]any); ok && mcMap["enabled"] != nil {+ nodeEnabled = parserConfigTruthy(mcMap["enabled"])+ } else if params["enable_metadata"] != nil {+ nodeEnabled = parserConfigTruthy(params["enable_metadata"])+ } else if enableMetadata {+ nodeEnabled = true+ }++ if len(nodeCustom) > 0 || len(nodeBuiltIn) > 0 || nodeEnabled {+ params["metadata"] = map[string]any{+ "enabled": nodeEnabled,+ "metadata": nodeCustom,+ "built_in_metadata": nodeBuiltIn,+ }+ } else {+ params["metadata"] = map[string]any{+ "enabled": false,+ "metadata": []any{},+ "built_in_metadata": []any{},+ }+ }}+ delete(params, "metadata_config")case strings.HasPrefix(cpnLower, "compiler:") || strings.HasPrefix(cpnLower, "compiler_"):if value, _ := params["llm_id"].(string); strings.TrimSpace(value) == "" && strings.TrimSpace(llmID) != "" {params["llm_id"] = llmID@@ -51,20 +93,66 @@ func ApplyComponentScopedParserConfig(}func mergeMetadataFields(parserConfig entity.JSONMap) []any {- var out []any- for _, key := range []string{"metadata", "built_in_metadata"} {- for _, item := range anySlice(parserConfig[key]) {- field, ok := item.(map[string]any)- if !ok {- continue+ custom := extractFieldList(parserConfig, "fields", "metadata")+ builtIn := extractFieldList(parserConfig, "built_in_metadata", "")+ out := make([]any, 0, len(custom)+len(builtIn))+ out = append(out, custom...)+ out = append(out, builtIn...)+ return out+}++func extractFieldList(parserConfig entity.JSONMap, primaryKey, fallbackKey string) []any {+ if parserConfig == nil {+ return []any{}+ }+ if raw, ok := parserConfig[primaryKey]; ok && raw != nil {+ if slice := extractValidFields(raw); len(slice) > 0 || primaryKey == "fields" {+ return slice+ }+ }+ if fallbackKey != "" {+ if raw, ok := parserConfig[fallbackKey]; ok && raw != nil {+ if slice := extractValidFields(raw); len(slice) > 0 {+ return slice}- name, _ := field["key"].(string)- if strings.TrimSpace(name) == "" {- continue+ }+ }+ for _, subKey := range []string{"metadata", "metadata_config"} {+ if metaMap, ok := parserConfig[subKey].(map[string]any); ok {+ if raw, ok := metaMap[primaryKey]; ok && raw != nil {+ if slice := extractValidFields(raw); len(slice) > 0 {+ return slice+ }+ }+ if fallbackKey != "" {+ if raw, ok := metaMap[fallbackKey]; ok && raw != nil {+ if slice := extractValidFields(raw); len(slice) > 0 {+ return slice+ }+ }}- out = append(out, field)}}+ return []any{}+}++func extractValidFields(value any) []any {+ items := anySlice(value)+ if items == nil {+ return []any{}+ }+ out := make([]any, 0, len(items))+ for _, item := range items {+ field, ok := item.(map[string]any)+ if !ok {+ continue+ }+ name, _ := field["key"].(string)+ if strings.TrimSpace(name) == "" {+ continue+ }+ out = append(out, field)+ }return out}@@ -78,6 +166,27 @@ func anySlice(value any) []any {out = append(out, item)}return out+ case []map[string]string:+ out := make([]any, 0, len(typed))+ for _, item := range typed {+ m := make(map[string]any, len(item))+ for k, v := range item {+ m[k] = v+ }+ out = append(out, m)+ }+ return out+ case []MetadataConfigField:+ out := make([]any, 0, len(typed))+ for _, item := range typed {+ out = append(out, map[string]any{+ "key": item.Key,+ "type": item.Type,+ "description": item.Description,+ "enum": item.Enum,+ })+ }+ return outdefault:return nil}@@ -90,11 +199,17 @@ func hasMetadataConfigShape(parserConfig entity.JSONMap) bool {if _, ok := parserConfig["enable_metadata"]; ok {return true}- for _, key := range []string{"metadata", "built_in_metadata"} {+ if _, ok := parserConfig["enabled"]; ok {+ return true+ }+ for _, key := range []string{"fields", "metadata", "built_in_metadata"} {if anySlice(parserConfig[key]) != nil {return true}}+ if _, ok := parserConfig["metadata"].(map[string]any); ok {+ return true+ }return false}diff --git a/internal/service/component_scoped_parser_config_test.go b/internal/service/component_scoped_parser_config_test.goindex bce355d2be..0ffb3a6248 100644--- a/internal/service/component_scoped_parser_config_test.go+++ b/internal/service/component_scoped_parser_config_test.go@@ -29,15 +29,20 @@ func TestApplyComponentScopedParserConfig_SyncsExtractorAndCompiler(t *testing.Tif extractor["llm_id"] != "llm-default" {t.Fatalf("extractor llm_id = %#v, want llm-default", extractor["llm_id"])}- if extractor["enable_metadata"] != 1 {- t.Fatalf("extractor enable_metadata = %#v, want 1", extractor["enable_metadata"])+ if _, ok := extractor["enable_metadata"]; ok {+ t.Fatalf("extractor enable_metadata should not be present, got %#v", extractor["enable_metadata"])}- wantFields := []any{- map[string]any{"key": "author", "type": "string"},- map[string]any{"key": "document_name", "type": "string"},+ wantMetadata := map[string]any{+ "enabled": true,+ "metadata": []any{+ map[string]any{"key": "author", "type": "string"},+ },+ "built_in_metadata": []any{+ map[string]any{"key": "document_name", "type": "string"},+ },}- if !reflect.DeepEqual(extractor["metadata"], wantFields) {- t.Fatalf("extractor metadata = %#v, want %#v", extractor["metadata"], wantFields)+ if !reflect.DeepEqual(extractor["metadata"], wantMetadata) {+ t.Fatalf("extractor metadata = %#v, want %#v", extractor["metadata"], wantMetadata)}compiler := got["Compiler:KnownSwiftLions"].(map[string]any)@@ -84,12 +89,17 @@ func TestApplyComponentScopedParserConfig_AcceptsTypedMetadataSlices(t *testing.got := ApplyComponentScopedParserConfig(parserConfig, "tenant-llm")extractor := got["Extractor:AutoExtractDefault"].(map[string]any)- wantFields := []any{- map[string]interface{}{"key": "author", "type": "string"},- map[string]interface{}{"key": "document_name", "type": "string"},+ wantMetadata := map[string]any{+ "enabled": true,+ "metadata": []any{+ map[string]interface{}{"key": "author", "type": "string"},+ },+ "built_in_metadata": []any{+ map[string]interface{}{"key": "document_name", "type": "string"},+ },}- if !reflect.DeepEqual(extractor["metadata"], wantFields) {- t.Fatalf("extractor metadata = %#v, want %#v", extractor["metadata"], wantFields)+ if !reflect.DeepEqual(extractor["metadata"], wantMetadata) {+ t.Fatalf("extractor metadata = %#v, want %#v", extractor["metadata"], wantMetadata)}}@@ -102,8 +112,11 @@ func TestApplyComponentScopedParserConfig_ClearsExtractorMetadataWhenDisabled(t},"Extractor:AutoExtractDefault": map[string]any{"enable_metadata": 1,- "metadata": []any{- map[string]any{"key": "stale", "type": "string"},+ "metadata": map[string]any{+ "enabled": true,+ "metadata": []any{+ map[string]any{"key": "stale", "type": "string"},+ },},},}@@ -111,11 +124,18 @@ func TestApplyComponentScopedParserConfig_ClearsExtractorMetadataWhenDisabled(tgot := ApplyComponentScopedParserConfig(parserConfig, "tenant-llm")extractor := got["Extractor:AutoExtractDefault"].(map[string]any)- if extractor["enable_metadata"] != 0 {- t.Fatalf("extractor enable_metadata = %#v, want 0", extractor["enable_metadata"])+ if _, ok := extractor["enable_metadata"]; ok {+ t.Fatalf("extractor enable_metadata should not be present, got %#v", extractor["enable_metadata"])}- if !reflect.DeepEqual(extractor["metadata"], []any{}) {- t.Fatalf("extractor metadata = %#v, want empty list", extractor["metadata"])+ wantMetadata := map[string]any{+ "enabled": false,+ "metadata": []any{},+ "built_in_metadata": []any{+ map[string]interface{}{"key": "document_name", "type": "string"},+ },+ }+ if !reflect.DeepEqual(extractor["metadata"], wantMetadata) {+ t.Fatalf("extractor metadata = %#v, want %#v", extractor["metadata"], wantMetadata)}}@@ -124,24 +144,117 @@ func TestApplyComponentScopedParserConfig_DoesNotTreatDocumentMetadataValuesAsSc"metadata": map[string]interface{}{"author": "Alice",},- "Extractor:AutoExtractDefault": map[string]any{- "enable_metadata": 1,- "metadata": []any{- map[string]any{"key": "author", "type": "string"},- },- },+ "Extractor:AutoExtractDefault": map[string]any{},}got := ApplyComponentScopedParserConfig(parserConfig, "tenant-llm")extractor := got["Extractor:AutoExtractDefault"].(map[string]any)- if extractor["enable_metadata"] != 1 {- t.Fatalf("extractor enable_metadata = %#v, want 1", extractor["enable_metadata"])+ wantMetadata := map[string]any{+ "enabled": false,+ "metadata": []any{},+ "built_in_metadata": []any{},+ }+ if !reflect.DeepEqual(extractor["metadata"], wantMetadata) {+ t.Fatalf("extractor metadata = %#v, want %#v", extractor["metadata"], wantMetadata)+ }+}++func TestMergeMetadataFields_MergesFieldsAndBuiltIn(t *testing.T) {+ parserConfig := entity.JSONMap{+ "metadata": []any{+ map[string]any{"key": "author", "type": "string"},+ },+ "built_in_metadata": []any{+ map[string]any{"key": "document_name", "type": "string"},+ },}++ got := mergeMetadataFields(parserConfig)want := []any{map[string]any{"key": "author", "type": "string"},+ map[string]any{"key": "document_name", "type": "string"},+ }+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("mergeMetadataFields = %#v, want %#v", got, want)+ }+}++func TestApplyComponentScopedParserConfig_PreservesNodeOnlyMetadataOnDatasetSave(t *testing.T) {+ parserConfig := entity.JSONMap{+ "Extractor:CanvasNode": map[string]any{+ "llm_id": "model-1",+ "metadata": map[string]any{+ "enabled": true,+ "metadata": []any{+ map[string]any{"key": "node_author", "type": "string"},+ },+ "built_in_metadata": []any{+ map[string]any{"key": "node_doc_title", "type": "string"},+ },+ },+ },+ }++ got := ApplyComponentScopedParserConfig(parserConfig, "tenant-llm")+ node := got["Extractor:CanvasNode"].(map[string]any)++ meta, ok := node["metadata"].(map[string]any)+ if !ok {+ t.Fatalf("expected node metadata map, got %T: %#v", node["metadata"], node["metadata"])+ }+ if enabled, _ := meta["enabled"].(bool); !enabled {+ t.Fatalf("expected node metadata.enabled = true, got false")+ }++ fields, ok := meta["metadata"].([]any)+ if !ok || len(fields) != 1 {+ t.Fatalf("expected 1 custom field preserved, got %#v", meta["metadata"])+ }+ if fields[0].(map[string]any)["key"] != "node_author" {+ t.Fatalf("expected node_author preserved, got %#v", fields[0])+ }+ builtIn, ok := meta["built_in_metadata"].([]any)+ if !ok || len(builtIn) != 1 {+ t.Fatalf("expected 1 built-in field preserved, got %#v", meta["built_in_metadata"])+ }+ if builtIn[0].(map[string]any)["key"] != "node_doc_title" {+ t.Fatalf("expected node_doc_title preserved, got %#v", builtIn[0])+ }+}++func TestApplyComponentScopedParserConfig_PreservesNodeOnlyMetadataConfigShape(t *testing.T) {+ parserConfig := entity.JSONMap{+ "enable_metadata": false,+ "Extractor:CanvasNode": map[string]any{+ "llm_id": "model-1",+ "metadata_config": map[string]any{+ "enabled": true,+ "metadata": []any{+ map[string]any{"key": "custom_field", "type": "string"},+ },+ },+ },+ }++ got := ApplyComponentScopedParserConfig(parserConfig, "tenant-llm")+ node := got["Extractor:CanvasNode"].(map[string]any)++ if _, ok := node["metadata_config"]; ok {+ t.Fatalf("metadata_config should be deleted from node, got %#v", node["metadata_config"])+ }+ meta, ok := node["metadata"].(map[string]any)+ if !ok {+ t.Fatalf("expected node metadata map, got %T: %#v", node["metadata"], node["metadata"])+ }+ if enabled, _ := meta["enabled"].(bool); !enabled {+ t.Fatalf("expected node metadata.enabled = true, got false")+ }+ fields, ok := meta["metadata"].([]any)+ if !ok || len(fields) != 1 {+ t.Fatalf("expected 1 custom field preserved, got %#v", meta["metadata"])}- if !reflect.DeepEqual(extractor["metadata"], want) {- t.Fatalf("extractor metadata = %#v, want %#v", extractor["metadata"], want)+ if fields[0].(map[string]any)["key"] != "custom_field" {+ t.Fatalf("expected custom_field preserved, got %#v", fields[0])}}diff --git a/internal/service/dataset/helpers.go b/internal/service/dataset/helpers.goindex 1dee37048b..28aac5e863 100644--- a/internal/service/dataset/helpers.go+++ b/internal/service/dataset/helpers.go@@ -253,7 +253,7 @@ func preserveDatasetParserConfigMetadata(next, existing entity.JSONMap, incomingif next == nil {next = entity.JSONMap{}}- for _, key := range []string{"metadata", "built_in_metadata", "enable_metadata"} {+ for _, key := range []string{"fields", "metadata", "built_in_metadata", "enable_metadata"} {if incoming != nil {if value, ok := incoming[key]; ok {next[key] = valuediff --git a/internal/service/dataset/metadata.go b/internal/service/dataset/metadata.goindex 49b25d0e0e..13c929fee0 100644--- a/internal/service/dataset/metadata.go+++ b/internal/service/dataset/metadata.go@@ -77,8 +77,10 @@ func (d *DatasetService) GetMetadataConfig(ctx context.Context, datasetID, tenanreturn nil, common.CodeDataError, fmt.Errorf("user '%s' lacks permission for dataset '%s'", tenantID, datasetID)}+ metadata := parserConfigValueOrEmptyList(kb.ParserConfig, "metadata")+return map[string]interface{}{- "metadata": parserConfigValueOrEmptyList(kb.ParserConfig, "metadata"),+ "metadata": metadata,"built_in_metadata": parserConfigValueOrEmptyList(kb.ParserConfig, "built_in_metadata"),}, common.CodeSuccess, nil}@@ -122,6 +124,18 @@ func (d *DatasetService) UpdateMetadataConfig(ctx context.Context, datasetID, te}parserConfig["metadata"] = metadataparserConfig["built_in_metadata"] = builtInMetadata+ // enable_metadata state machine:+ // - If req.Enabled is explicitly set: use *req.Enabled.+ // - If nil (uninitialized dataset): auto-enable if metadata fields exist.+ // - If false (explicitly disabled by user): preserve false even when fields are added.+ // - If fields are empty: auto-disable to avoid invoking extractor on empty schema.+ if req.Enabled != nil {+ parserConfig["enable_metadata"] = *req.Enabled+ } else if parserConfig["enable_metadata"] == nil {+ parserConfig["enable_metadata"] = len(metadata) > 0 || len(builtInMetadata) > 0+ } else if len(metadata) == 0 && len(builtInMetadata) == 0 {+ parserConfig["enable_metadata"] = false+ }if tenant, tenantErr := d.tenantDAO.GetByID(ctx, dao.DB, kb.TenantID); tenantErr == nil && tenant != nil {parserConfig = service.ApplyComponentScopedParserConfig(parserConfig,diff --git a/internal/service/dataset/metadata_config_test.go b/internal/service/dataset/metadata_config_test.goindex ddc0f3f248..e5074c251b 100644--- a/internal/service/dataset/metadata_config_test.go+++ b/internal/service/dataset/metadata_config_test.go@@ -235,9 +235,11 @@ func TestDatasetServiceUpdateMetadataConfigSyncsExtractorSchema(t *testing.T) {Update("parser_config", entity.JSONMap{"enable_metadata": false,"Extractor:AutoExtractDefault": map[string]any{- "enable_metadata": 1,- "metadata": []any{- map[string]any{"key": "stale", "type": "string"},+ "metadata": map[string]any{+ "enabled": true,+ "fields": []any{+ map[string]any{"key": "stale", "type": "string"},+ },},},}).Error; err != nil {@@ -274,10 +276,18 @@ func TestDatasetServiceUpdateMetadataConfigSyncsExtractorSchema(t *testing.T) {if !ok {t.Fatalf("expected extractor component params, got %#v", persisted.ParserConfig["Extractor:AutoExtractDefault"])}- if got := metadataFlagInt(t, extractor["enable_metadata"]); got != 0 {- t.Fatalf("extractor enable_metadata = %#v, want 0 when top-level flag stays disabled", extractor["enable_metadata"])+ if _, ok := extractor["enable_metadata"]; ok {+ t.Fatalf("extractor enable_metadata should not be present, got %#v", extractor["enable_metadata"])+ }+ extractorMeta, ok := extractor["metadata"].(map[string]interface{})+ if !ok {+ t.Fatalf("expected modular metadata map in extractor, got %#v", extractor["metadata"])+ }+ if enabled, ok := extractorMeta["enabled"].(bool); !ok || enabled {+ t.Fatalf("extractor metadata.enabled = %#v, want false when top-level flag stays disabled", extractorMeta["enabled"])}+ // State 2: When enabled, updating fields updates component with enabled = trueif err := dao.DB.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Update("parser_config", entity.JSONMap{@@ -313,27 +323,100 @@ func TestDatasetServiceUpdateMetadataConfigSyncsExtractorSchema(t *testing.T) {if !ok {t.Fatalf("expected extractor component params, got %#v", persisted.ParserConfig["Extractor:AutoExtractDefault"])}- if got := metadataFlagInt(t, extractor["enable_metadata"]); got != 1 {- t.Fatalf("extractor enable_metadata = %#v, want 1", extractor["enable_metadata"])+ extractorMeta, ok = extractor["metadata"].(map[string]interface{})+ if !ok {+ t.Fatalf("expected modular metadata map in extractor, got %#v", extractor["metadata"])+ }+ if enabled, ok := extractorMeta["enabled"].(bool); !ok || !enabled {+ t.Fatalf("extractor metadata.enabled = %#v, want true", extractorMeta["enabled"])+ }+ gotFields, ok := extractorMeta["metadata"].([]interface{})+ if !ok {+ t.Fatalf("extractor metadata.metadata = %#v, want []interface{}", extractorMeta["metadata"])+ }+ if len(gotFields) != 1 {+ t.Fatalf("extractor metadata.metadata len = %d, want 1 (%#v)", len(gotFields), gotFields)}- gotFields, ok := extractor["metadata"].([]interface{})+ gotBuiltIn, ok := extractorMeta["built_in_metadata"].([]interface{})if !ok {- t.Fatalf("extractor metadata = %#v, want []interface{}", extractor["metadata"])- }- wantFields := []map[string]interface{}{- {"key": "author", "type": "string"},- {"key": "document_name", "type": "string"},- }- if len(gotFields) != len(wantFields) {- t.Fatalf("extractor metadata len = %d, want %d (%#v)", len(gotFields), len(wantFields), gotFields)- }- for i, want := range wantFields {- field, ok := gotFields[i].(map[string]interface{})- if !ok {- t.Fatalf("extractor metadata[%d] = %#v, want map[string]interface{}", i, gotFields[i])- }- if field["key"] != want["key"] || field["type"] != want["type"] {- t.Fatalf("extractor metadata[%d] = %#v, want key/type %#v", i, field, want)- }+ t.Fatalf("extractor metadata.built_in_metadata = %#v, want []interface{}", extractorMeta["built_in_metadata"])+ }+ if len(gotBuiltIn) != 1 {+ t.Fatalf("extractor metadata.built_in_metadata len = %d, want 1 (%#v)", len(gotBuiltIn), gotBuiltIn)+ }++ // State 3: Empty fields auto-disables enable_metadata+ _, code, err = (&DatasetService{+ kbDAO: dao.NewKnowledgebaseDAO(),+ tenantDAO: dao.NewTenantDAO(),+ }).UpdateMetadataConfig(ctx, "kb-1", "tenant-1", &service.MetadataConfigRequest{+ Metadata: []service.MetadataConfigField{},+ BuiltInMetadata: []service.MetadataConfigField{},+ })+ if err != nil {+ t.Fatalf("UpdateMetadataConfig with empty fields failed: %v", err)+ }+ if code != common.CodeSuccess {+ t.Fatalf("expected success code, got %d", code)+ }+ persisted, err = dao.NewKnowledgebaseDAO().GetByID(ctx, db, "kb-1")+ if err != nil {+ t.Fatalf("failed to fetch persisted dataset: %v", err)+ }+ if enabled, ok := persisted.ParserConfig["enable_metadata"].(bool); !ok || enabled {+ t.Fatalf("expected parser_config.enable_metadata == false after emptying fields, got %#v", persisted.ParserConfig["enable_metadata"])+ }+}++func TestDatasetServiceGetMetadataConfig(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertCreateDatasetTenant(t, "tenant-1")+ insertDatasetMetadataConfigKB(t, "kb-1", "tenant-1")++ ctx := t.Context()+ _, code, err := (&DatasetService{+ kbDAO: dao.NewKnowledgebaseDAO(),+ tenantDAO: dao.NewTenantDAO(),+ }).UpdateMetadataConfig(ctx, "kb-1", "tenant-1", &service.MetadataConfigRequest{+ Metadata: []service.MetadataConfigField{+ {Key: "author", Type: "string"},+ },+ BuiltInMetadata: []service.MetadataConfigField{+ {Key: "file_name", Type: "string"},+ },+ })+ if err != nil {+ t.Fatalf("UpdateMetadataConfig failed: %v", err)+ }+ if code != common.CodeSuccess {+ t.Fatalf("expected success code, got %d", code)+ }++ result, code, err := (&DatasetService{+ kbDAO: dao.NewKnowledgebaseDAO(),+ tenantDAO: dao.NewTenantDAO(),+ }).GetMetadataConfig(ctx, "kb-1", "tenant-1")+ if err != nil {+ t.Fatalf("GetMetadataConfig failed: %v", err)+ }+ if code != common.CodeSuccess {+ t.Fatalf("expected success code, got %d", code)+ }+ metadata, ok := result["metadata"].([]interface{})+ if !ok || len(metadata) != 1 {+ t.Fatalf("expected 1 metadata item, got %#v", result["metadata"])+ }+ field0, ok := metadata[0].(map[string]interface{})+ if !ok || field0["key"] != "author" {+ t.Fatalf("expected key author, got %#v", metadata[0])+ }+ builtIn, ok := result["built_in_metadata"].([]interface{})+ if !ok || len(builtIn) != 1 {+ t.Fatalf("expected 1 built_in_metadata item, got %#v", result["built_in_metadata"])+ }+ builtIn0, ok := builtIn[0].(map[string]interface{})+ if !ok || builtIn0["key"] != "file_name" {+ t.Fatalf("expected key file_name, got %#v", builtIn[0])}}diff --git a/internal/service/dataset_types.go b/internal/service/dataset_types.goindex 6d61253eba..a1be4a7280 100644--- a/internal/service/dataset_types.go+++ b/internal/service/dataset_types.go@@ -115,6 +115,7 @@ type MetadataConfigField struct {// MetadataConfigRequest mirrors PUT /datasets/:dataset_id/metadata/config.type MetadataConfigRequest struct {+ Enabled *bool `json:"enabled,omitempty"`Metadata []MetadataConfigField `json:"metadata"`BuiltInMetadata []MetadataConfigField `json:"built_in_metadata"`}