Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(extractor): unify 5-in-1 modular schema across dataset and pipeline, drop legacy field_name and custom prompt (#18432)

jay777212a1a93ed3abf
diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.go
index 417c61d9ff..63b11a7b43 100644
--- a/internal/ingestion/component/extractor.go
+++ b/internal/ingestion/component/extractor.go
@@ -14,50 +14,20 @@
// limitations under the License.
//
-// Package component — Extractor component (Phase 2.5 of
-// port-rag-flow-pipeline-to-go.md §4 row 2.5).
+// Package component — Extractor component.
//
-// SCOPE (honest):
+// SCOPE:
//
-// - PROVIDER-AGNOSTIC (§8 Q1): the Extractor does NOT depend on any
+// - PROVIDER-AGNOSTIC: the Extractor does NOT depend on any
// specific LLM provider. It dispatches every chat call through
-// internal/entity/models — the same factory routes 48 of the 56
-// Python ChatModel providers registered there (factory.go switch,
-// lines 36-156). The 8 providers NOT yet in the Go switch (LeptonAI,
-// Gemini LiteLLM path, PerfXCloud, 01.AI / Lingyi, DeerAPI,
-// Astraflow-CN, RAGcon, New API) ARE unreachable from this
-// component — an llm_id resolving to one of those falls through to
-// NewDummyModel and the chat call returns a deterministic "dummy"
-// response. We DO NOT panic: errors are surfaced as a clean
-// "no driver for %q" wrap that callers can log and route.
+// internal/entity/models.
//
-// - LLM CALL SHAPE: one chat call per chunk (no batching). LLM
-// calls are inherently serial; sequential per-chunk processing
-// keeps test ordering deterministic under -race.
+// - CONCURRENCY: auto extraction tasks (keywords, questions, summary, metadata)
+// run concurrently across chunks via a bounded worker pool (extractorPool).
//
// - TIMEOUT / ELAPSED: the call is wrapped in
-// runtime.WithTimeout(60s) and runtime.TrackElapsed so the
-// upstream pipeline gets _created_time / _elapsed_time stamps
-// matching the python ProcessBase contract (base.py:42, 58).
-//
-// - JSON PARSING: the prompt asks the LLM to return a JSON object;
-// we best-effort parse the response into map[string]any. A
-// non-JSON response is NOT a hard error — it's surfaced as the
-// raw string under the same field name so downstream callers
-// can decide what to do.
-//
-// - WHAT IS NOT YET PORTED: the python _build_TOC branch
-// (rag/flow/extractor/extractor.py:40-72) requires the TOC
-// generator (rag.prompts.generator.run_toc_from_text). That
-// service has no Go counterpart yet; the current Extractor
-// short-circuits with a clear error when field_name == "toc"
-// so a future Phase 2.5+ task can fill the gap without a
-// silent regression.
-//
-// - SINGLE-CHUNK FAST PATH: when no chunk list is wired in,
-// the LLM is called once with the resolved args directly (no
-// chunk substitution). Matches python _invoke path
-// (line 108: msg, sys_prompt = self._sys_prompt_and_msg([], args)).
+// runtime.WithTimeout(600s) and runtime.TrackElapsed so the
+// upstream pipeline gets _created_time / _elapsed_time stamps.
package component
import (
@@ -167,12 +137,7 @@ Extract the most important keywords/phrases of a given piece of text content.
- Summarize the text content, and give the top %d important keywords/phrases.
- The keywords MUST be in the same language as the given piece of text content.
- The keywords are delimited by ENGLISH COMMA.
-- Output keywords ONLY.
-
----
-
-## Text Content
-%s`
+- Output keywords ONLY.`
autoQuestionPrompt = `## Role
You are a text analyzer.
@@ -186,12 +151,7 @@ Propose questions about a given piece of text content.
- The questions SHOULD cover the main content of the text as much as possible.
- The questions MUST be in the same language as the given piece of text content.
- One question per line.
-- Output questions ONLY.
-
----
-
-## Text Content
-%s`
+- Output questions ONLY.`
autoMetadataPrompt = `## Role: Metadata extraction expert.
## Rules:
@@ -203,9 +163,6 @@ Propose questions about a given piece of text content.
- Output: ONLY a valid JSON string. No Markdown, no notes.
## Schema for extraction:
-%s
-
-## Content to analyze:
%s`
autoSummaryPrompt = `## Role
@@ -239,10 +196,12 @@ type ExtractorComponent struct {
// Param map shape (all keys optional; missing → Defaults()):
//
// {
-// "field_name": string, — optional; key the extraction lands under
// "llm_id": string, — optional; resolves via models.NewModelFactory
-// "system_prompt": string, — optional override
-// "prompt": string, — optional user prompt
+// "keywords": map[string]any, — optional auto keywords extraction config
+// "questions": map[string]any, — optional auto questions extraction config
+// "tags": map[string]any, — optional auto tags extraction config
+// "summary": map[string]any, — optional auto summary extraction config
+// "metadata": map[string]any, — optional auto metadata extraction config
// }
//
// errors here surface as canvas compile failures so a malformed
@@ -250,70 +209,31 @@ type ExtractorComponent struct {
func NewExtractorComponent(params map[string]any) (runtime.Component, error) {
p := schema.ExtractorParam{}.Defaults()
if params != nil {
- if v, ok := params["field_name"].(string); ok {
- p.FieldName = v
- }
if v, ok := params["llm_id"].(string); ok {
p.LLMID = v
}
- if v, ok := params["system_prompt"].(string); ok {
- p.SystemPrompt = v
- } else if v, ok := params["sys_prompt"].(string); ok {
- p.SystemPrompt = v
- }
- if v, ok := params["prompt"].(string); ok {
- p.Prompt = v
- } else if v, ok := params["prompts"].(string); ok && v != "" {
- // Python agent/component/llm.py:119-120 normalizes a bare-string
- // prompts into [{"role":"user","content":prompts}]. Mirror that
- // here so a front-end/template that emits prompts as a string
- // (the graph.nodes form / dsl testdata) is not silently dropped
- // by the .([]any) assertion on the list branch below.
- p.Prompt = v
- } else if promptsRaw, ok := params["prompts"].([]any); ok && len(promptsRaw) > 0 {
- if first, ok := promptsRaw[0].(map[string]any); ok {
- if content, ok := first["content"].(string); ok {
- p.Prompt = content
- }
- }
- }
- // 1. Keywords (modular or flat)
+
+ // 1. Keywords
if kwRaw, ok := params["keywords"].(map[string]any); ok {
if v, ok := kwRaw["top_n"]; ok {
p.Keywords.TopN = mapInt(v)
}
if v, ok := kwRaw["system_prompt"].(string); ok {
p.Keywords.SystemPrompt = v
- } else if v, ok := kwRaw["sys_prompt"].(string); ok {
- p.Keywords.SystemPrompt = v
- }
- } else if v, ok := params["auto_keywords"]; ok {
- p.Keywords.TopN = mapInt(v)
- if v, ok := params["keywords_sys_prompt"].(string); ok {
- p.Keywords.SystemPrompt = v
}
}
- p.AutoKeywords = p.Keywords.TopN
- // 2. Questions (modular or flat)
+ // 2. Questions
if qRaw, ok := params["questions"].(map[string]any); ok {
if v, ok := qRaw["top_n"]; ok {
p.Questions.TopN = mapInt(v)
}
if v, ok := qRaw["system_prompt"].(string); ok {
p.Questions.SystemPrompt = v
- } else if v, ok := qRaw["sys_prompt"].(string); ok {
- p.Questions.SystemPrompt = v
- }
- } else if v, ok := params["auto_questions"]; ok {
- p.Questions.TopN = mapInt(v)
- if v, ok := params["questions_sys_prompt"].(string); ok {
- p.Questions.SystemPrompt = v
}
}
- p.AutoQuestions = p.Questions.TopN
- // 3. Tags (modular or flat)
+ // 3. Tags
if tagRaw, ok := params["tags"].(map[string]any); ok {
if v, ok := tagRaw["top_n"]; ok {
p.Tags.TopN = mapInt(v)
@@ -321,18 +241,9 @@ func NewExtractorComponent(params map[string]any) (runtime.Component, error) {
if v, ok := tagRaw["tag_file_id"].(string); ok {
p.Tags.TagFileID = v
}
- } else {
- if v, ok := params["auto_tags"]; ok {
- p.Tags.TopN = mapInt(v)
- }
- if v, ok := params["tag_file_id"].(string); ok {
- p.Tags.TagFileID = v
- }
}
- p.AutoTags = p.Tags.TopN
- p.TagFileID = p.Tags.TagFileID
- // 4. Summary (modular or flat)
+ // 4. Summary
if sumRaw, ok := params["summary"].(map[string]any); ok {
if v, ok := sumRaw["enabled"].(bool); ok {
p.Summary.Enabled = v
@@ -341,72 +252,22 @@ func NewExtractorComponent(params map[string]any) (runtime.Component, error) {
}
if v, ok := sumRaw["system_prompt"].(string); ok {
p.Summary.SystemPrompt = v
- } else if v, ok := sumRaw["sys_prompt"].(string); ok {
- p.Summary.SystemPrompt = v
- }
- } else {
- if v, ok := params["enable_summary"].(bool); ok {
- p.Summary.Enabled = v
- } else if v, ok := params["enable_summary"]; ok {
- p.Summary.Enabled = mapInt(v) == 1
- } else if p.FieldName == "summary" {
- p.Summary.Enabled = true
- }
- p.Summary.SystemPrompt = p.SystemPrompt
- }
- if p.Summary.Enabled {
- p.EnableSummary = 1
- if p.FieldName == "" {
- p.FieldName = "summary"
- }
- if p.Summary.SystemPrompt != "" && p.SystemPrompt == "" {
- p.SystemPrompt = p.Summary.SystemPrompt
- }
- } else {
- p.EnableSummary = 0
- // When summary is explicitly disabled, avoid legacy "summary" field_name default triggering open-ended calls
- if p.FieldName == "summary" {
- p.FieldName = ""
}
}
- // Metadata sub-config parsing with legacy flat fallback
- if metaRaw, ok := params["metadata_config"].(map[string]any); ok {
+ // 5. Metadata
+ if metaRaw, ok := params["metadata"].(map[string]any); ok {
if v, ok := metaRaw["enabled"].(bool); ok {
- p.MetadataConfig.Enabled = v
+ p.Metadata.Enabled = v
} else if v, ok := metaRaw["enabled"]; ok {
- p.MetadataConfig.Enabled = mapInt(v) == 1
+ p.Metadata.Enabled = mapInt(v) == 1
}
if v, ok := metaRaw["metadata"]; ok {
- p.MetadataConfig.Metadata = parseMetadataFieldDefs(v)
+ p.Metadata.Metadata = parseMetadataFieldDefs(v)
}
if v, ok := metaRaw["built_in_metadata"]; ok {
- p.MetadataConfig.BuiltInMetadata = parseMetadataFieldDefs(v)
- }
- if p.MetadataConfig.Enabled {
- p.EnableMetadata = 1
- } else {
- p.EnableMetadata = 0
- }
- p.Metadata = p.MetadataConfig.Metadata
- } else {
- if v, ok := params["enable_metadata"].(bool); ok {
- if v {
- p.EnableMetadata = 1
- } else {
- p.EnableMetadata = 0
- }
- } else if v, ok := params["enable_metadata"]; ok {
- p.EnableMetadata = mapInt(v)
- }
- if v, ok := params["metadata"]; ok {
- p.Metadata = parseMetadataFieldDefs(v)
- }
- if v, ok := params["built_in_metadata"]; ok {
- p.MetadataConfig.BuiltInMetadata = parseMetadataFieldDefs(v)
+ p.Metadata.BuiltInMetadata = parseMetadataFieldDefs(v)
}
- p.MetadataConfig.Enabled = p.EnableMetadata > 0
- p.MetadataConfig.Metadata = p.Metadata
}
}
if err := p.Validate(); err != nil {
@@ -421,21 +282,17 @@ func NewExtractorComponent(params map[string]any) (runtime.Component, error) {
// self.chat_mdl; the Go port exposes it explicitly).
func (c *ExtractorComponent) Inputs() map[string]string {
return map[string]string{
- "chunks": "List of map[string]any from upstream Tokenizer. Each entry must carry a string 'text' (or 'content_with_weight') field. Optional — when absent the LLM is called once with the resolved args.",
- "prompt": "Optional user prompt template. Falls back to Param.Prompt when absent.",
- "llm_id": "Optional per-call LLM id override. Falls back to Param.LLMID when absent.",
- "system_prompt": "Optional per-call system prompt override. Falls back to Param.SystemPrompt.",
+ "chunks": "List of map[string]any from upstream Tokenizer. Each entry must carry a string 'text' (or 'content_with_weight') field. Optional — when absent the LLM is called once with the resolved args.",
+ "llm_id": "Optional per-call LLM id override. Falls back to Param.LLMID when absent.",
}
}
// Outputs returns the public surface downstream ingestion
// consumers can wire into. Mirrors schema.ExtractorOutputs.
//
-// chunks []map[string]any — input chunks, each augmented
-// with field_name=<LLM result>.
-// When the input chunks list is
-// absent, the slice contains a
-// single map with the same shape.
+// chunks []map[string]any — input chunks, each augmented with
+// extracted modular fields (important_kwd,
+// question_kwd, tag_kwd, summary, metadata).
// output_format string — always "chunks". Parity with
// python set_output contract.
// _ERROR string — populated on a short-circuit
@@ -443,7 +300,7 @@ func (c *ExtractorComponent) Inputs() map[string]string {
// set_output("_ERROR", ...)).
func (c *ExtractorComponent) Outputs() map[string]string {
return map[string]string{
- "chunks": "Extraction results — input chunks (or a single-element slice when no chunks were supplied), each enriched with field_name=<LLM response>.",
+ "chunks": "Extraction results — input chunks, each enriched with modular extraction fields (important_kwd, question_kwd, tag_kwd, summary, metadata).",
"output_format": "Always \"chunks\". Parity marker for downstream consumers.",
"_ERROR": "Optional short-circuit error message (reserved for the future TOC branch and other error paths).",
}
@@ -632,16 +489,12 @@ func toExtractorEinoMessages(msgs []eschema.Message) []*eschema.Message {
// input map. Computed once at the top of Invoke so the rest of
// the function reads as straight-line code.
type extractorInputs struct {
- fieldName string
- llmID string
- systemPrompt string
- prompt string
- lang string
- chunks []map[string]any
+ llmID string
+ lang string
+ chunks []map[string]any
// temperature overrides the LLM temperature for this call. A
// nil value leaves the request's Temperature unset so the model
- // (or the chat-model default) decides, matching Python's generic
- // Extractor path. The keyword/question helpers set it to
+ // (or the chat-model default) decides. The keyword/question helpers set it to
// extractorTemperature (0.2) to mirror generator.py.
temperature *float64
}
@@ -649,16 +502,10 @@ type extractorInputs struct {
// resolveInputs overlays per-call inputs on top of the
// component's static Param. Missing keys fall back to the
// Param-level values; per-call values win on conflict (so a
-// canvas can override LLM_ID at runtime). The python
-// Extractor reads inputs directly from get_input_elements(); the
-// Go port normalizes to extractorInputs once at the top so the
-// rest of Invoke reads straight-line.
+// canvas can override LLM_ID at runtime).
func (c *ExtractorComponent) resolveInputs(inputs map[string]any) extractorInputs {
out := extractorInputs{
- fieldName: c.Param.FieldName,
- llmID: c.Param.LLMID,
- systemPrompt: c.Param.SystemPrompt,
- prompt: c.Param.Prompt,
+ llmID: c.Param.LLMID,
}
if inputs == nil {
return out
@@ -666,14 +513,6 @@ func (c *ExtractorComponent) resolveInputs(inputs map[string]any) extractorInput
if v, ok := inputs["llm_id"].(string); ok && v != "" {
out.llmID = v
}
- if v, ok := inputs["prompt"].(string); ok && v != "" {
- out.prompt = v
- }
- if v, ok := inputs["system_prompt"].(string); ok && v != "" {
- out.systemPrompt = v
- } else if v, ok := inputs["sys_prompt"].(string); ok && v != "" {
- out.systemPrompt = v
- }
if v, ok := inputs["lang"].(string); ok && v != "" {
out.lang = v
}
@@ -728,16 +567,11 @@ func extractorChunkList(v any) ([]map[string]any, bool) {
//
// chunks (optional, []map[string]any) — upstream chunks; each must
// carry a string "text".
-// prompt (optional, string) — overrides Param.Prompt.
-// system_prompt (optional, string) — overrides Param.SystemPrompt.
// llm_id (optional, string) — overrides Param.LLMID.
//
// Outputs:
//
-// chunks ([]map[string]any) — input chunks augmented with
-// field_name=<LLM result>. When
-// the input list is empty, the
-// slice contains a single map.
+// chunks ([]map[string]any) — input chunks augmented with extraction results.
// output_format (string) — always "chunks".
// _ERROR (string, reserved) — populated when the component
// short-circuits with an error.
@@ -752,13 +586,16 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map
zap.String("component", "Extractor"),
zap.Int("input_chunks", len(in.chunks)),
)
- if in.fieldName == "toc" {
- return nil, fmt.Errorf("extractor: field_name %q requires the TOC prompt generator which is not yet ported to Go", "toc")
+ if len(in.chunks) == 0 {
+ return map[string]any{
+ "chunks": []map[string]any{},
+ "output_format": "chunks",
+ }, nil
}
if err := runtime.WithTimeout(ctx, extractorTimeout, func(timeoutCtx context.Context) error {
- // Tag phase: run when auto_tags > 0 and we have chunks.
- if c.Param.AutoTags > 0 && len(in.chunks) > 0 {
+ // Tag phase: run when tags.top_n > 0 and we have chunks.
+ if c.Param.Tags.TopN > 0 {
tagged, tagErr := c.runAutoTags(timeoutCtx, db, in)
if tagErr != nil {
return tagErr
@@ -766,62 +603,7 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map
in.chunks = tagged
}
- if len(in.chunks) == 0 {
- // Render the prompt with an empty chunk map so body placeholders
- // resolve (or fall back to appending nothing) before the LLM call.
- // Without this, a template containing {text} would be forwarded
- // to the model unsubstituted.
- callIn := in
- if in.fieldName == "summary" && strings.TrimSpace(callIn.systemPrompt) == "" && strings.TrimSpace(c.Param.Summary.SystemPrompt) == "" {
- callIn.systemPrompt = autoSummaryPrompt
- }
- callIn.systemPrompt, callIn.prompt = renderExtractorPrompts(
- callIn.systemPrompt, in.prompt, map[string]any{}, "",
- )
- ans, callErr := c.callText(timeoutCtx, db, callIn, "")
- if callErr != nil {
- return callErr
- }
- ck := map[string]any{}
- ck[in.fieldName] = ans
- in.chunks = []map[string]any{ck}
- return nil
- }
- // Auto keyword / question / metadata extraction runs with
- // cross-chunk concurrency through the process-wide extractor
- // pool (mirrors Python's chat_limiter). Each chunk job owns its
- // own chunk map, so no per-chunk mutex is required; per-invocation
- // completion is tracked inside runAutoExtractions.
- if err := c.runAutoExtractions(timeoutCtx, db, in); err != nil {
- return err
- }
-
- // Primary field extraction stays sequential per chunk: each
- // chunk's result lands under its own field_name key, preserving
- // deterministic output ordering (see file header: -race tests).
- for i, ck := range in.chunks {
- if in.fieldName == "" {
- continue
- }
- text, _ := ck["content_with_weight"].(string)
- if strings.TrimSpace(text) == "" {
- text, _ = ck["text"].(string)
- }
- // Substitute {field_name} placeholders with current
- // chunk field values, mirroring Python's
- // string_format at extractor.py:103.
- callIn := in
- if in.fieldName == "summary" && strings.TrimSpace(callIn.systemPrompt) == "" && strings.TrimSpace(c.Param.Summary.SystemPrompt) == "" {
- callIn.systemPrompt = autoSummaryPrompt
- }
- callIn.systemPrompt, callIn.prompt = renderExtractorPrompts(callIn.systemPrompt, in.prompt, ck, text)
- ans, callErr := c.callText(timeoutCtx, db, callIn, "")
- if callErr != nil {
- return fmt.Errorf("chunk %d: %w", i, callErr)
- }
- ck[in.fieldName] = ans
- }
- return nil
+ return c.runAutoExtractions(timeoutCtx, db, in)
}); err != nil {
return nil, fmt.Errorf("extractor: %w", err)
}
@@ -836,35 +618,26 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map
}
// runAutoKeywords extracts keywords for the current chunk and stores
-// them on ck["important_kwd"]. mu may be nil (sequential path); when
-// non-nil it serializes the shared chunk-map accesses so concurrent
-// keyword/question goroutines stay race-free. The existence check and
-// the map writes are both guarded by mu. Keyword extraction pins
+// them on ck["important_kwd"]. Keyword extraction pins
// temperature to extractorTemperature (0.2) to mirror generator.py.
func (c *ExtractorComponent) runAutoKeywords(ctx context.Context, db *gorm.DB, in extractorInputs, ck map[string]any, chunkText string) error {
- _, exists := ck["important_kwd"]
- if exists {
+ if _, exists := ck["important_kwd"]; exists {
return nil
}
topN := c.Param.Keywords.TopN
if topN <= 0 {
- topN = c.Param.AutoKeywords
+ return nil
}
- var sysPrompt, userPrompt string
- if strings.TrimSpace(c.Param.Keywords.SystemPrompt) != "" {
- sysPrompt, userPrompt = renderExtractorPrompts(c.Param.Keywords.SystemPrompt, "Output: ", ck, chunkText)
- } else {
- sysPrompt = fmt.Sprintf(autoKeywordPrompt, topN, chunkText)
- userPrompt = "Output: "
+ systemPrompt := c.Param.Keywords.SystemPrompt
+ if strings.TrimSpace(systemPrompt) == "" {
+ systemPrompt = fmt.Sprintf(autoKeywordPrompt, topN)
}
kwTemp := extractorTemperature
kwIn := extractorInputs{
- llmID: in.llmID,
- systemPrompt: sysPrompt,
- prompt: userPrompt,
- temperature: &kwTemp,
+ llmID: in.llmID,
+ temperature: &kwTemp,
}
- resultStr, err := c.callText(ctx, db, kwIn, "")
+ resultStr, err := c.callText(ctx, db, kwIn, systemPrompt, chunkText)
if err != nil {
return err
}
@@ -888,29 +661,23 @@ func (c *ExtractorComponent) runAutoKeywords(ctx context.Context, db *gorm.DB, i
// runAutoQuestions extracts questions for the current chunk and stores
// them on ck["question_kwd"]. See runAutoKeywords for the temperature pin.
func (c *ExtractorComponent) runAutoQuestions(ctx context.Context, db *gorm.DB, in extractorInputs, ck map[string]any, chunkText string) error {
- _, exists := ck["question_kwd"]
- if exists {
+ if _, exists := ck["question_kwd"]; exists {
return nil
}
topN := c.Param.Questions.TopN
if topN <= 0 {
- topN = c.Param.AutoQuestions
+ return nil
}
- var sysPrompt, userPrompt string
- if strings.TrimSpace(c.Param.Questions.SystemPrompt) != "" {
- sysPrompt, userPrompt = renderExtractorPrompts(c.Param.Questions.SystemPrompt, "Output: ", ck, chunkText)
- } else {
- sysPrompt = fmt.Sprintf(autoQuestionPrompt, topN, chunkText)
- userPrompt = "Output: "
+ systemPrompt := c.Param.Questions.SystemPrompt
+ if strings.TrimSpace(systemPrompt) == "" {
+ systemPrompt = fmt.Sprintf(autoQuestionPrompt, topN)
}
qTemp := extractorTemperature
qIn := extractorInputs{
- llmID: in.llmID,
- systemPrompt: sysPrompt,
- prompt: userPrompt,
- temperature: &qTemp,
+ llmID: in.llmID,
+ temperature: &qTemp,
}
- resultStr, err := c.callText(ctx, db, qIn, "")
+ resultStr, err := c.callText(ctx, db, qIn, systemPrompt, chunkText)
if err != nil {
return err
}
@@ -939,7 +706,37 @@ func (c *ExtractorComponent) runAutoQuestions(ctx context.Context, db *gorm.DB,
return nil
}
-// runAutoExtractions dispatches the auto keyword / question / metadata
+// runAutoSummary extracts a concise summary for the current chunk using autoSummaryPrompt
+// and stores it on ck["summary"].
+func (c *ExtractorComponent) runAutoSummary(ctx context.Context, db *gorm.DB, in extractorInputs, ck map[string]any, chunkText string) error {
+ if _, exists := ck["summary"]; exists {
+ return nil
+ }
+ if !c.Param.Summary.Enabled {
+ return nil
+ }
+ systemPrompt := c.Param.Summary.SystemPrompt
+ if strings.TrimSpace(systemPrompt) == "" {
+ systemPrompt = autoSummaryPrompt
+ }
+ sumTemp := extractorTemperature
+ sumIn := extractorInputs{
+ llmID: in.llmID,
+ temperature: &sumTemp,
+ }
+ resultStr, err := c.callText(ctx, db, sumIn, systemPrompt, chunkText)
+ if err != nil {
+ return err
+ }
+ resultStr = cleanExtractionResult(resultStr)
+ if resultStr == "" {
+ return nil
+ }
+ ck["summary"] = resultStr
+ return nil
+}
+
+// runAutoExtractions dispatches the auto keyword / question / summary / metadata
// extraction for every chunk to the process-wide extractorPool so the
// work runs with bounded cross-chunk concurrency (mirrors Python's
// chat_limiter). The pool only bounds concurrency; per-invocation
@@ -947,7 +744,7 @@ func (c *ExtractorComponent) runAutoQuestions(ctx context.Context, db *gorm.DB,
// sync.WaitGroup, so concurrent Invoke calls do not disturb each other.
// Each chunk job owns its own chunk map, so no per-chunk mutex is needed.
func (c *ExtractorComponent) runAutoExtractions(ctx context.Context, db *gorm.DB, in extractorInputs) error {
- if c.Param.AutoKeywords == 0 && c.Param.AutoQuestions == 0 && c.Param.EnableMetadata == 0 {
+ if c.Param.Keywords.TopN <= 0 && c.Param.Questions.TopN <= 0 && !c.Param.Summary.Enabled && !c.Param.Metadata.Enabled {
return nil
}
futs := make([]utility.WorkerPoolFuture[extractorJob, struct{}], 0, len(in.chunks))
@@ -994,23 +791,28 @@ func (c *ExtractorComponent) runAutoExtractions(ctx context.Context, db *gorm.DB
}
// autoExtractionJob returns the unit of work for one chunk: it runs the
-// enabled auto-extractions (keywords, questions, metadata) sequentially
+// enabled auto-extractions (keywords, questions, summary, metadata) sequentially
// on the chunk's own map. Running them sequentially inside the job keeps
// the code free of per-chunk mutexes — cross-chunk parallelism is provided
// by the pool, not by intra-chunk goroutines.
func (c *ExtractorComponent) autoExtractionJob(ctx context.Context, db *gorm.DB, in extractorInputs, idx int, ck map[string]any, chunkText string) extractorJob {
return func() error {
- if c.Param.AutoKeywords > 0 {
+ if c.Param.Keywords.TopN > 0 {
if err := c.runAutoKeywords(ctx, db, in, ck, chunkText); err != nil {
return fmt.Errorf("chunk %d keywords: %w", idx, err)
}
}
- if c.Param.AutoQuestions > 0 {
+ if c.Param.Questions.TopN > 0 {
if err := c.runAutoQuestions(ctx, db, in, ck, chunkText); err != nil {
return fmt.Errorf("chunk %d questions: %w", idx, err)
}
}
- if c.Param.EnableMetadata > 0 {
+ if c.Param.Summary.Enabled {
+ if err := c.runAutoSummary(ctx, db, in, ck, chunkText); err != nil {
+ return fmt.Errorf("chunk %d summary: %w", idx, err)
+ }
+ }
+ if c.Param.Metadata.Enabled {
if err := c.runEnableMetadata(ctx, db, in, ck, chunkText); err != nil {
return fmt.Errorf("chunk %d metadata: %w", idx, err)
}
@@ -1022,15 +824,14 @@ func (c *ExtractorComponent) autoExtractionJob(ctx context.Context, db *gorm.DB,
// runEnableMetadata extracts structured metadata for the current chunk and
// merges the parsed JSON object into ck["metadata"]. It mirrors the
// runAutoKeywords/runAutoQuestions shape but parses a JSON object and
-// merges into the chunk's metadata map (which the existing
-// mergeChunkMetadata → docState.apply aggregation path consumes).
+// merges into the chunk's metadata map.
func (c *ExtractorComponent) runEnableMetadata(ctx context.Context, db *gorm.DB, in extractorInputs, ck map[string]any, chunkText string) error {
- if len(c.Param.Metadata) == 0 {
+ if !c.Param.Metadata.Enabled || len(c.Param.Metadata.Metadata) == 0 {
return nil
}
// Render the field schema into the prompt, mirroring Python's
// turn2jsonschema(metadata_conf) rendered into the META_DATA template.
- schemaMap := common.Turn2JSONSchema(c.Param.Metadata)
+ schemaMap := common.Turn2JSONSchema(c.Param.Metadata.Metadata)
if len(schemaMap) == 0 {
return nil
}
@@ -1052,19 +853,16 @@ func (c *ExtractorComponent) runEnableMetadata(ctx context.Context, db *gorm.DB,
} else {
metaTemp := extractorTemperature
metaIn := extractorInputs{
- llmID: in.llmID,
- systemPrompt: fmt.Sprintf(autoMetadataPrompt, schemaStr, chunkText),
- prompt: "Output: ",
- temperature: &metaTemp,
+ llmID: in.llmID,
+ temperature: &metaTemp,
}
- parsed, err = c.callStructured(ctx, db, metaIn, "")
+ systemPrompt := fmt.Sprintf(autoMetadataPrompt, schemaStr)
+ parsed, err = c.callStructured(ctx, db, metaIn, systemPrompt, chunkText)
if err != nil {
return err
}
if parsed == nil {
// Non-JSON or empty response — nothing to extract, not an error.
- // Matches Python gen_metadata treating an unparseable reply as an
- // empty result rather than failing the chunk.
return nil
}
setMetadataLLMCache(ctx, in.llmID, schemaStr, chunkText, parsed)
@@ -1138,6 +936,58 @@ func setMetadataLLMCache(ctx context.Context, llmID, schemaJSON, chunkText strin
client.Set(ctx, metadataLLMCacheKey(llmID, schemaJSON, chunkText), string(data), metadataLLMCacheTTL)
}
+// callRaw runs one chat call against the LLM (per chunk in the normal path)
+// and returns the raw response. It holds the shared plumbing — driver/target
+// resolution, message assembly, temperature override, retry — but deliberately
+// does NOT clean or parse the response. Callers pick the view they need:
+//
+// - callText wraps callRaw with the LLM-layer two-step cleanup (think +
+// tool_call) and returns a plain string.
+// - callStructured wraps callRaw with cleanup + explicit JSON parsing and
+// returns a map — the metadata path, matching Python gen_metadata.
+func (c *ExtractorComponent) callRaw(ctx context.Context, db *gorm.DB, in extractorInputs, systemPrompt, chunkText string) (*extractorChatResponse, error) {
+ driver, modelName, apiKey, baseURL, err := resolveExtractorChatTarget(ctx, db, in.llmID)
+ if err != nil {
+ return nil, err
+ }
+ msgs := buildExtractorMessages(systemPrompt, chunkText)
+ fitted, fitErr := fitExtractorMessages(ctx, db, in.llmID, msgs)
+ if fitErr != nil {
+ return nil, fitErr
+ }
+ msgs = fitted
+ inv := getExtractorChatInvoker()
+ req := extractorChatRequest{
+ Driver: driver,
+ ModelName: modelName,
+ APIKey: apiKey,
+ BaseURL: baseURL,
+ Messages: msgs,
+ }
+ // Only override the temperature when the caller set one. A nil
+ // Temperature lets the model / chat-model default decide, matching
+ // Python's generic Extractor path; keyword/question helpers set
+ // extractorTemperature (0.2) to mirror generator.py.
+ if in.temperature != nil {
+ temp := *in.temperature
+ req.Temperature = &temp
+ }
+ var resp *extractorChatResponse
+ if err := common.RetryWithBackoff(ctx, extractorRetryMax, extractorRetryDelay, func() error {
+ r, e := inv.Chat(ctx, req)
+ resp = r
+ return e
+ }, isRetryableLLMError); err != nil {
+ return nil, err
+ }
+ if resp == nil {
+ // Defensive: a provider adapter returning (nil, nil) would otherwise
+ // panic on resp.Content below. Surface a diagnosable error instead.
+ return nil, fmt.Errorf("extractor: chat: nil response from invoker")
+ }
+ return resp, nil
+}
+
// toolCallRE matches a <tool_call>...</tool_call> block, mirroring Python's
// `re.sub(r"<tool_call>.*?</tool_call>", "", txt, flags=re.DOTALL)` at
// llm_service.py:461. Non-greedy so consecutive tool_call blocks are each
@@ -1236,72 +1086,9 @@ func isRetryableLLMError(err error) bool {
return true
}
-// callRaw dispatches one LLM chat call for the supplied chunk text (empty
-// string in the no-chunk fast path) and returns the raw response. It holds
-// the shared plumbing — driver/target resolution, message assembly,
-// temperature override, retry — but deliberately does NOT clean or parse the
-// response. Callers pick the view they need:
-//
-// - callText wraps callRaw with the LLM-layer two-step cleanup (think +
-// tool_call) and returns a plain string — the field-extraction path,
-// matching Python _generate_async.
-// - callStructured wraps callRaw with cleanup + explicit JSON parsing and
-// returns a map — the metadata path, matching Python gen_metadata.
-//
-// Splitting this way restores Python's separation of concerns: the LLM layer
-// (async_chat) cleans, the caller decides whether to parse. A single
-// all-purpose call() that best-effort parses would hand field extraction a
-// map it does not want (silently dropped downstream) while leaving metadata
-// to re-parse — and could not add cleaning without breaking JSON structure.
-func (c *ExtractorComponent) callRaw(ctx context.Context, db *gorm.DB, in extractorInputs, chunkText string) (*extractorChatResponse, error) {
- driver, modelName, apiKey, baseURL, err := resolveExtractorChatTarget(ctx, db, in.llmID)
- if err != nil {
- return nil, err
- }
- msgs := buildExtractorMessages(in.systemPrompt, in.prompt)
- fitted, fitErr := fitExtractorMessages(ctx, db, in.llmID, msgs)
- if fitErr != nil {
- return nil, fitErr
- }
- msgs = fitted
- inv := getExtractorChatInvoker()
- req := extractorChatRequest{
- Driver: driver,
- ModelName: modelName,
- APIKey: apiKey,
- BaseURL: baseURL,
- Messages: msgs,
- }
- // Only override the temperature when the caller set one. A nil
- // Temperature lets the model / chat-model default decide, matching
- // Python's generic Extractor path; keyword/question helpers set
- // extractorTemperature (0.2) to mirror generator.py.
- if in.temperature != nil {
- temp := *in.temperature
- req.Temperature = &temp
- }
- var resp *extractorChatResponse
- if err := common.RetryWithBackoff(ctx, extractorRetryMax, extractorRetryDelay, func() error {
- r, e := inv.Chat(ctx, req)
- resp = r
- return e
- }, isRetryableLLMError); err != nil {
- return nil, err
- }
- if resp == nil {
- // Defensive: a provider adapter returning (nil, nil) would otherwise
- // panic on resp.Content below. Surface a diagnosable error instead.
- return nil, fmt.Errorf("extractor: chat: nil response from invoker")
- }
- return resp, nil
-}
-
-// callText runs one chat call and returns the cleaned text response. Used by
-// the field-extraction path (empty-chunks fast path and per-chunk writes),
-// which must always produce a string — matching Python's _generate_async
-// (llm.py:374-377) returning a raw string with no JSON parsing.
-func (c *ExtractorComponent) callText(ctx context.Context, db *gorm.DB, in extractorInputs, chunkText string) (string, error) {
- resp, err := c.callRaw(ctx, db, in, chunkText)
+// callText runs one chat call and returns the cleaned text response.
+func (c *ExtractorComponent) callText(ctx context.Context, db *gorm.DB, in extractorInputs, systemPrompt, chunkText string) (string, error) {
+ resp, err := c.callRaw(ctx, db, in, systemPrompt, chunkText)
if err != nil {
return "", err
}
@@ -1313,8 +1100,8 @@ func (c *ExtractorComponent) callText(ctx context.Context, db *gorm.DB, in extra
// structured value to merge into chunk metadata — matching Python
// gen_metadata's json_repair.loads. A non-JSON or empty response returns
// (nil, nil) so the caller treats it as "nothing extracted", not an error.
-func (c *ExtractorComponent) callStructured(ctx context.Context, db *gorm.DB, in extractorInputs, chunkText string) (map[string]any, error) {
- resp, err := c.callRaw(ctx, db, in, chunkText)
+func (c *ExtractorComponent) callStructured(ctx context.Context, db *gorm.DB, in extractorInputs, systemPrompt, chunkText string) (map[string]any, error) {
+ resp, err := c.callRaw(ctx, db, in, systemPrompt, chunkText)
if err != nil {
return nil, err
}
@@ -1644,107 +1431,21 @@ func fitExtractorMessages(ctx context.Context, db *gorm.DB, llmID string, msgs [
}
// buildExtractorMessages assembles system + user messages for one extraction
-// call. Prompt rendering (placeholder substitution, chunk-text injection, and
-// empty-prompt normalization) is performed upstream by renderExtractorPrompts;
-// this function is a pure structural assembler with no rendering logic.
-func buildExtractorMessages(system, user string) []eschema.Message {
+// call. The user message strictly carries chunkText (or a fallback single space
+// if empty), ensuring a clean and consistent message contract.
+// System prompt is omitted if empty or whitespace-only to avoid sending empty
+// system turns to LLM providers.
+func buildExtractorMessages(systemPrompt, chunkText string) []eschema.Message {
out := make([]eschema.Message, 0, 2)
- if system != "" {
- out = append(out, eschema.Message{Role: eschema.System, Content: system})
- }
- out = append(out, eschema.Message{Role: eschema.User, Content: user})
- return out
-}
-
-// unifiedPlaceholderRE matches plain field placeholders {fieldName} and
-// canvas macro placeholders {ComponentName:ParamName@fieldName}.
-var unifiedPlaceholderRE = regexp.MustCompile(`\{([A-Za-z0-9_]+(:[A-Za-z0-9_]+)?@[A-Za-z0-9_]+|[A-Za-z_][A-Za-z0-9_]*)\}`)
-
-// bodyPlaceholderAliases lists plain placeholder names that resolve to chunk body content.
-var bodyPlaceholderAliases = map[string]bool{
- "text": true,
- "chunks": true,
- "content_with_weight": true,
-}
-
-// isBodyPlaceholder reports whether key represents chunk body content
-// (including upstream output references such as @chunks, @text, @markdown).
-func isBodyPlaceholder(key string) bool {
- if bodyPlaceholderAliases[key] {
- return true
- }
- return strings.HasSuffix(key, "@chunks") ||
- strings.HasSuffix(key, "@text") ||
- strings.HasSuffix(key, "@markdown")
-}
-
-// renderExtractorPrompts performs single-pass rendering of system and user
-// prompt templates for one chunk:
-// 1. Content placeholders ({text}, {chunks}, {content_with_weight},
-// {ComponentName:ParamName@chunks}, etc.) are resolved: first from the chunk
-// map, falling back to chunkText. If a non-empty value is resolved,
-// bodyInjected is marked true.
-// 2. Chunk metadata fields (such as {title}, {author}) are resolved from the
-// chunk map.
-// 3. Unrecognized placeholders are preserved verbatim.
-// 4. If no non-empty chunk body was injected anywhere in the system or user
-// prompt, chunkText is automatically appended to the user prompt as a fallback.
-func renderExtractorPrompts(sysTemplate, userTemplate string, ck map[string]any, chunkText string) (string, string) {
- var bodyInjected bool
-
- render := func(tmpl string) string {
- if tmpl == "" {
- return ""
- }
- return unifiedPlaceholderRE.ReplaceAllStringFunc(tmpl, func(match string) string {
- key := match[1 : len(match)-1] // strip { }
-
- // A. Content placeholders: check chunk map first, fallback to chunkText
- if isBodyPlaceholder(key) {
- var val string
- if raw, ok := ck[key]; ok {
- val = fmt.Sprintf("%v", raw)
- }
- if strings.TrimSpace(val) == "" {
- val = chunkText
- }
- if strings.TrimSpace(val) != "" {
- bodyInjected = true
- }
- return val
- }
-
- // B. Chunk metadata fields
- if val, ok := ck[key]; ok {
- return fmt.Sprintf("%v", val)
- }
-
- // C. Leave unknown placeholders as-is
- return match
- })
- }
-
- renderedSys := render(sysTemplate)
- renderedUser := render(userTemplate)
-
- if !bodyInjected && strings.TrimSpace(chunkText) != "" {
- // TrimSpace before testing emptiness is intentional: a user template
- // that renders to pure whitespace (e.g. " ") must not produce a
- // leading "\n\n" separator — the result should be chunkText alone.
- // A plain `if renderedUser != ""` check would fail that invariant.
- trimmed := strings.TrimSpace(renderedUser)
- if trimmed != "" {
- renderedUser = trimmed + "\n\n" + chunkText
- } else {
- renderedUser = chunkText
- }
+ if strings.TrimSpace(systemPrompt) != "" {
+ out = append(out, eschema.Message{Role: eschema.System, Content: systemPrompt})
}
-
- if strings.TrimSpace(renderedUser) == "" {
- renderedUser = " "
+ userContent := chunkText
+ if strings.TrimSpace(userContent) == "" {
+ userContent = " "
}
-
- return renderedSys, renderedUser
+ out = append(out, eschema.Message{Role: eschema.User, Content: userContent})
+ return out
}
// tryParseJSONObject tries to parse s as a JSON object. Returns
@@ -1801,8 +1502,22 @@ func tryParseJSONObject(s string) (map[string]any, bool) {
// parseMetadataFieldDefs converts an any value (typically []any of maps)
// to a typed []common.MetadataFieldDef slice.
func parseMetadataFieldDefs(v any) []common.MetadataFieldDef {
- arr, ok := v.([]any)
- if !ok {
+ if v == nil {
+ return nil
+ }
+ if defs, ok := v.([]common.MetadataFieldDef); ok {
+ return defs
+ }
+ var arr []any
+ switch typed := v.(type) {
+ case []any:
+ arr = typed
+ case []map[string]any:
+ arr = make([]any, 0, len(typed))
+ for _, item := range typed {
+ arr = append(arr, item)
+ }
+ default:
return nil
}
fields := make([]common.MetadataFieldDef, 0, len(arr))
diff --git a/internal/ingestion/component/extractor_tag.go b/internal/ingestion/component/extractor_tag.go
index a84b759a41..ec82e16a69 100644
--- a/internal/ingestion/component/extractor_tag.go
+++ b/internal/ingestion/component/extractor_tag.go
@@ -155,7 +155,7 @@ func (c *ExtractorComponent) runAutoTags(ctx context.Context, db *gorm.DB, in ex
return in.chunks, nil
}
- topN := c.Param.AutoTags
+ topN := c.Param.Tags.TopN
var examples []schema.TaggedChunk
var docsToTag []map[string]any
@@ -218,22 +218,22 @@ func (c *ExtractorComponent) runAutoTags(ctx context.Context, db *gorm.DB, in ex
}
func (c *ExtractorComponent) resolveTagSource(ctx context.Context) (*indexedTagSource, bool) {
- if c.Param.TagFileID == "" {
+ if c.Param.Tags.TagFileID == "" {
return nil, false
}
return c.loadTagFileIndexed(ctx)
}
func (c *ExtractorComponent) loadTagFileIndexed(ctx context.Context) (*indexedTagSource, bool) {
- f, err := dao.NewFileDAO().GetByID(ctx, dao.DB, c.Param.TagFileID)
+ f, err := dao.NewFileDAO().GetByID(ctx, dao.DB, c.Param.Tags.TagFileID)
if err != nil || f == nil || f.Location == nil || *f.Location == "" {
- common.Warn(fmt.Sprintf("extractor tags: resolve tag_file_id %q: %v", c.Param.TagFileID, err))
+ common.Warn(fmt.Sprintf("extractor tags: resolve tag_file_id %q: %v", c.Param.Tags.TagFileID, err))
return nil, false
}
cacheKey := tagSourceFileCacheKey(f)
if cached, ok := tagSourceFileIndexCache.load(cacheKey); ok {
common.Info("extractor tags: reused tag source file index",
- zap.String("file_id", c.Param.TagFileID),
+ zap.String("file_id", c.Param.Tags.TagFileID),
zap.String("bucket", f.ParentID),
zap.String("key", *f.Location),
)
@@ -256,7 +256,7 @@ func (c *ExtractorComponent) loadTagFileIndexed(ctx context.Context) (*indexedTa
}
indexed = tagSourceFileIndexCache.store(cacheKey, indexed)
common.Info("extractor tags: loaded tag source file",
- zap.String("file_id", c.Param.TagFileID),
+ zap.String("file_id", c.Param.Tags.TagFileID),
zap.String("bucket", f.ParentID),
zap.String("key", *f.Location),
zap.Int64("size", f.Size),
diff --git a/internal/ingestion/component/extractor_tag_test.go b/internal/ingestion/component/extractor_tag_test.go
index 9e7c8df374..a5260bb77e 100644
--- a/internal/ingestion/component/extractor_tag_test.go
+++ b/internal/ingestion/component/extractor_tag_test.go
@@ -45,7 +45,7 @@ func pushExtractorTagTargetResolverStub(t *testing.T) {
func TestExtractorTags_NoTagFileID(t *testing.T) {
pushExtractorTagChatStub(t, nil)
- comp, _ := NewExtractorComponent(map[string]any{"auto_tags": 3})
+ comp, _ := NewExtractorComponent(map[string]any{"tags": map[string]any{"top_n": 3}})
out, err := comp.Invoke(t.Context(), nil, map[string]any{
"chunks": []map[string]any{
{"content_with_weight": "test"},
@@ -65,7 +65,7 @@ func TestExtractorTags_NoTagFileID(t *testing.T) {
func TestExtractorTags_NoLLMID(t *testing.T) {
pushExtractorTagChatStub(t, nil)
- comp, _ := NewExtractorComponent(map[string]any{"auto_tags": 3})
+ comp, _ := NewExtractorComponent(map[string]any{"tags": map[string]any{"top_n": 3}})
out, err := comp.Invoke(t.Context(), nil, map[string]any{
"chunks": []map[string]any{
{"content_with_weight": "some unrelated text"},
@@ -85,9 +85,13 @@ func TestExtractorTags_WithKeywords(t *testing.T) {
pushExtractorTagTargetResolverStub(t)
comp, _ := NewExtractorComponent(map[string]any{
- "llm_id": "test@test",
- "auto_tags": 3,
- "auto_keywords": 3,
+ "llm_id": "test@test",
+ "tags": map[string]any{
+ "top_n": 3,
+ },
+ "keywords": map[string]any{
+ "top_n": 3,
+ },
})
out, err := comp.Invoke(t.Context(), nil, map[string]any{
"chunks": []map[string]any{
diff --git a/internal/ingestion/component/extractor_test.go b/internal/ingestion/component/extractor_test.go
index fd738a0a0e..e5aff9f85b 100644
--- a/internal/ingestion/component/extractor_test.go
+++ b/internal/ingestion/component/extractor_test.go
@@ -102,8 +102,7 @@ func withStubChatInvoker(t *testing.T, responses ...stubResponse) *stubExtractor
}
// TestExtractorComponent_Registered verifies the init() registration
-// is visible to the runtime registry (Phase 4 / API layer
-// depends on this).
+// is visible to the runtime registry.
func TestExtractorComponent_Registered(t *testing.T) {
factory, cat, md, ok := runtime.DefaultRegistry.Lookup("Extractor")
if !ok {
@@ -130,8 +129,7 @@ func TestExtractorComponent_Registered(t *testing.T) {
}
// TestExtractorComponent_Invoke_HappyPath covers the per-chunk
-// fan-out: two chunks in → two LLM calls → each chunk enriched
-// with the field_name key.
+// auto-extraction (e.g. summary).
func TestExtractorComponent_Invoke_HappyPath(t *testing.T) {
withStubChatInvoker(t,
stubResponse{Content: "answer for chunk 1"},
@@ -139,9 +137,8 @@ func TestExtractorComponent_Invoke_HappyPath(t *testing.T) {
)
c := &ExtractorComponent{Param: schema.ExtractorParam{
- FieldName: "summary",
- LLMID: "gpt-4o-mini",
- Prompt: "Summarize:",
+ LLMID: "gpt-4o-mini",
+ Summary: schema.SummaryExtractConfig{Enabled: true},
}}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"chunks": []map[string]any{
@@ -160,11 +157,10 @@ func TestExtractorComponent_Invoke_HappyPath(t *testing.T) {
if len(chunks) != 2 {
t.Fatalf("chunks len = %d, want 2", len(chunks))
}
- if chunks[0]["summary"] != "answer for chunk 1" {
- t.Errorf("chunk[0].summary = %v, want %q", chunks[0]["summary"], "answer for chunk 1")
- }
- if chunks[1]["summary"] != "answer for chunk 2" {
- t.Errorf("chunk[1].summary = %v, want %q", chunks[1]["summary"], "answer for chunk 2")
+ s0, _ := chunks[0]["summary"].(string)
+ s1, _ := chunks[1]["summary"].(string)
+ if !((s0 == "answer for chunk 1" && s1 == "answer for chunk 2") || (s0 == "answer for chunk 2" && s1 == "answer for chunk 1")) {
+ t.Errorf("unexpected summaries: chunk0=%q, chunk1=%q", s0, s1)
}
if out["output_format"] != "chunks" {
t.Errorf("output_format = %v, want chunks", out["output_format"])
@@ -172,11 +168,8 @@ func TestExtractorComponent_Invoke_HappyPath(t *testing.T) {
}
// TestExtractorComponent_Invoke_LLMError verifies a mock LLM
-// error is surfaced through Invoke with the component-name prefix
-// so the upstream pipeline can attribute failures. After retry
-// (RetryWithBackoff: 3 retries), the error chains the cause.
+// error is surfaced through Invoke with the component-name prefix.
func TestExtractorComponent_Invoke_LLMError(t *testing.T) {
- // Fast retry for tests — avoid multi-second sleeps.
prevMax, prevDelay := extractorRetryMax, extractorRetryDelay
extractorRetryMax, extractorRetryDelay = 3, time.Millisecond
t.Cleanup(func() {
@@ -192,8 +185,8 @@ func TestExtractorComponent_Invoke_LLMError(t *testing.T) {
)
c := &ExtractorComponent{Param: schema.ExtractorParam{
- FieldName: "summary",
- LLMID: "gpt-4o-mini",
+ LLMID: "gpt-4o-mini",
+ Summary: schema.SummaryExtractConfig{Enabled: true},
}}
_, err := c.Invoke(t.Context(), nil, map[string]any{
"chunks": []map[string]any{{"text": "x"}},
@@ -226,8 +219,8 @@ func TestExtractorComponent_Invoke_RetrySucceeds(t *testing.T) {
)
c := &ExtractorComponent{Param: schema.ExtractorParam{
- FieldName: "summary",
- LLMID: "gpt-4o-mini",
+ LLMID: "gpt-4o-mini",
+ Summary: schema.SummaryExtractConfig{Enabled: true},
}}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"chunks": []map[string]any{{"text": "x"}},
@@ -246,49 +239,17 @@ func TestExtractorComponent_Invoke_RetrySucceeds(t *testing.T) {
// TestExtractorComponent_Invoke_UnknownProvider asserts the
// production (eino) chat invoker handles an unregistered driver
-// without panicking, per plan §8 Q1 ("48/56 providers covered;
-// the Extractor is provider-agnostic via llm_id; the 8 missing
-// are edge cases that do not block Phase 2.5").
-//
-// Design note: every other test in this file drives the
-// invoker through the production Component.Invoke path with a
-// canned-response invoker installed via SetExtractorChatInvoker
-// (the test seam). That seam accepts a pre-resolved driver
-// path; it cannot model the eino factory's default-branch
-// behaviour for an unknown driver. This test exercises the
-// production chat-invoker directly to pin that branch — the
-// production code path the real Extractor will hit when the
-// DSL references a provider that is not in the 48/56 covered
-// set.
-//
-// The contract under test:
-// - The call MUST NOT panic.
-// - On unknown driver, the factory's default branch routes to
-// a DummyModel that returns a deterministic error string
-// (we assert the error contains that sentinel so future
-// maintainers see the wiring goes through the factory,
-// not bypassed by a hand-rolled default).
+// without panicking.
func TestExtractorComponent_Invoke_UnknownProvider(t *testing.T) {
inv := &einoExtractorChatInvoker{}
resp, err := inv.Chat(context.Background(), extractorChatRequest{
Driver: "definitely-not-a-real-provider-xyz",
ModelName: "anything",
})
- // Either an error is returned OR a non-nil response is produced
- // by the DummyModel fallback. The contract is "no panic"; both
- // of these outcomes are acceptable. We only fail the test if
- // BOTH error and response are empty (which would indicate a
- // silent no-op).
if err == nil && resp == nil {
t.Fatal("production invoker returned nil error AND nil response for unknown driver — silent no-op")
}
- // When an error IS returned, it must mention the driver name so
- // operators can correlate the failure back to the DSL config.
if err != nil {
- // Acceptable error patterns for an unknown driver:
- // - mentions the driver name (correlatable for operators)
- // - "no driver"/"unknown" sentinels (typed error)
- // - "not implemented" (the eino dummy model fallback path)
if !strings.Contains(err.Error(), "definitely-not-a-real-provider-xyz") &&
!strings.Contains(err.Error(), "no driver") &&
!strings.Contains(err.Error(), "unknown") &&
@@ -298,120 +259,11 @@ func TestExtractorComponent_Invoke_UnknownProvider(t *testing.T) {
}
}
-// TestExtractorComponent_Invoke_KeepsJSONAsString verifies a JSON object
-// response from the LLM is written to the chunk's field_name value as a
-// plain string — matching Python's _generate_async, which returns the raw
-// string with no JSON parsing. (The Extractor does NOT parse field-extraction
-// results; only the metadata path, via callStructured, parses explicitly.)
-func TestExtractorComponent_Invoke_KeepsJSONAsString(t *testing.T) {
- withStubChatInvoker(t,
- stubResponse{Content: `{"answer": 42, "tags": ["a", "b"]}`},
- )
-
- c := &ExtractorComponent{Param: schema.ExtractorParam{
- FieldName: "extraction",
- Prompt: "extract:",
- }}
- out, err := c.Invoke(t.Context(), nil, map[string]any{
- "chunks": []map[string]any{{"text": "doc"}}},
- )
- if err != nil {
- t.Fatalf("Invoke: %v", err)
- }
- chunks := out["chunks"].([]map[string]any)
- got, ok := chunks[0]["extraction"].(string)
- if !ok {
- t.Fatalf("extraction should be a string (no JSON parse on field extraction), got %T", chunks[0]["extraction"])
- }
- if got != `{"answer": 42, "tags": ["a", "b"]}` {
- t.Errorf("extraction = %q, want the raw JSON string", got)
- }
-}
-
-// TestExtractorComponent_Invoke_KeepsJSONStringInFence verifies a JSON
-// response wrapped in a Markdown code fence is stored as the raw string —
-// the code fence is not stripped on the field-extraction path (Python's
-// _generate_async returns the raw text untouched).
-func TestExtractorComponent_Invoke_KeepsJSONStringInFence(t *testing.T) {
- withStubChatInvoker(t,
- stubResponse{Content: "```json\n{\"summary\": \"hello\"}\n```"},
- )
-
- c := &ExtractorComponent{Param: schema.ExtractorParam{
- FieldName: "out",
- }}
- out, err := c.Invoke(t.Context(), nil, map[string]any{
- "chunks": []map[string]any{{"text": "x"}}},
- )
- if err != nil {
- t.Fatalf("Invoke: %v", err)
- }
- got, ok := out["chunks"].([]map[string]any)[0]["out"].(string)
- if !ok {
- t.Fatalf("out should be a string, got %T", out["chunks"].([]map[string]any)[0]["out"])
- }
- if got != "```json\n{\"summary\": \"hello\"}\n```" {
- t.Errorf("out = %q, want the raw fenced JSON string", got)
- }
-}
-
-// TestExtractorComponent_Invoke_HandlesMalformedJSON verifies a
-// non-JSON response surfaces as the raw string under the
-// destination field — not an error. The python Extractor
-// accepts whatever the LLM emits; downstream callers decide
-// what to do with it.
-func TestExtractorComponent_Invoke_HandlesMalformedJSON(t *testing.T) {
- withStubChatInvoker(t,
- stubResponse{Content: "this is not JSON at all"},
- )
-
- c := &ExtractorComponent{Param: schema.ExtractorParam{
- FieldName: "raw",
- }}
- out, err := c.Invoke(t.Context(), nil, map[string]any{
- "chunks": []map[string]any{{"text": "x"}}},
- )
- if err != nil {
- t.Fatalf("Invoke returned error on non-JSON: %v", err)
- }
- got := out["chunks"].([]map[string]any)[0]["raw"]
- if got != "this is not JSON at all" {
- t.Errorf("raw = %v, want %q", got, "this is not JSON at all")
- }
-}
-
-// TestExtractorComponent_Invoke_TOCNotPorted asserts the
-// field_name=="toc" branch is gated by a clear error so a future
-// migration to the Go TOC generator doesn't accidentally fall
-// through to chunk iteration.
-func TestExtractorComponent_Invoke_TOCNotPorted(t *testing.T) {
+// TestExtractorComponent_Invoke_EmptyChunksReturnsEmpty verifies that
+// when len(in.chunks) == 0, Invoke immediately returns empty chunks.
+func TestExtractorComponent_Invoke_EmptyChunksReturnsEmpty(t *testing.T) {
c := &ExtractorComponent{Param: schema.ExtractorParam{
- FieldName: "toc",
- }}
- _, err := c.Invoke(t.Context(), nil, map[string]any{
- "chunks": []map[string]any{{"text": "x"}}},
- )
- if err == nil {
- t.Fatal("expected error for field_name=toc, got nil")
- }
- if !strings.Contains(err.Error(), "toc") {
- t.Errorf("error should mention toc: %v", err)
- }
- if !strings.Contains(err.Error(), "not yet ported") {
- t.Errorf("error should call out parity gap: %v", err)
- }
-}
-
-// TestExtractorComponent_Invoke_NoChunksFastPath verifies the
-// no-chunks input still produces a one-element chunks slice
-// (mirrors python _invoke line 110 fallback).
-func TestExtractorComponent_Invoke_NoChunksFastPath(t *testing.T) {
- withStubChatInvoker(t,
- stubResponse{Content: "single-shot answer"},
- )
-
- c := &ExtractorComponent{Param: schema.ExtractorParam{
- FieldName: "answer",
+ Summary: schema.SummaryExtractConfig{Enabled: true},
}}
out, err := c.Invoke(t.Context(), nil, map[string]any{})
if err != nil {
@@ -421,21 +273,20 @@ func TestExtractorComponent_Invoke_NoChunksFastPath(t *testing.T) {
if !ok {
t.Fatalf("chunks missing or wrong shape")
}
- if len(chunks) != 1 {
- t.Fatalf("chunks len = %d, want 1", len(chunks))
- }
- if chunks[0]["answer"] != "single-shot answer" {
- t.Errorf("answer = %v, want %q", chunks[0]["answer"], "single-shot answer")
+ if len(chunks) != 0 {
+ t.Fatalf("chunks len = %d, want 0", len(chunks))
}
}
+// TestExtractorComponent_Invoke_JSONListInput verifies that the chunks list
+// can be provided under the "json" key.
func TestExtractorComponent_Invoke_JSONListInput(t *testing.T) {
withStubChatInvoker(t,
- stubResponse{Content: "json chunk answer"},
+ stubResponse{Content: "json chunk summary"},
)
c := &ExtractorComponent{Param: schema.ExtractorParam{
- FieldName: "answer",
+ Summary: schema.SummaryExtractConfig{Enabled: true},
}}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"json": []map[string]any{{"text": "json payload chunk"}},
@@ -447,25 +298,24 @@ func TestExtractorComponent_Invoke_JSONListInput(t *testing.T) {
if !ok || len(chunks) != 1 {
t.Fatalf("chunks malformed: %v", out["chunks"])
}
- if chunks[0]["answer"] != "json chunk answer" {
- t.Errorf("answer = %v, want %q", chunks[0]["answer"], "json chunk answer")
+ if chunks[0]["summary"] != "json chunk summary" {
+ t.Errorf("summary = %v, want %q", chunks[0]["summary"], "json chunk summary")
}
}
// TestExtractorComponent_Invoke_PerCallLLMIDOverride verifies an
-// inputs["llm_id"] override wins over Param.LLMID and reaches
-// the chat invoker verbatim (the per-call override is the
-// explicit test seam for runtime reconfiguration).
+// inputs["llm_id"] override wins over Param.LLMID.
func TestExtractorComponent_Invoke_PerCallLLMIDOverride(t *testing.T) {
stub := withStubChatInvoker(t,
- stubResponse{Content: "ok"},
+ stubResponse{Content: "summary result"},
)
c := &ExtractorComponent{Param: schema.ExtractorParam{
- FieldName: "out",
- LLMID: "static-llm",
+ LLMID: "static-llm",
+ Summary: schema.SummaryExtractConfig{Enabled: true},
}}
_, err := c.Invoke(t.Context(), nil, map[string]any{
+ "chunks": []map[string]any{{"text": "sample text"}},
"llm_id": "override-llm",
})
if err != nil {
@@ -479,19 +329,18 @@ func TestExtractorComponent_Invoke_PerCallLLMIDOverride(t *testing.T) {
}
// TestExtractorComponent_Invoke_CompositeLLMID verifies the
-// composite "gpt-4o-mini@openai" form is split into driver and
-// model before reaching the chat invoker. Matches the canonical
-// composite llm_id convention used throughout the codebase
-// (see internal/agent/component/llm_credentials.go:parseLLMIDParts).
+// composite "gpt-4o-mini@openai" form is split into driver and model.
func TestExtractorComponent_Invoke_CompositeLLMID(t *testing.T) {
stub := withStubChatInvoker(t,
- stubResponse{Content: "ok"},
+ stubResponse{Content: "summary result"},
)
c := &ExtractorComponent{Param: schema.ExtractorParam{
- FieldName: "out",
- LLMID: "gpt-4o-mini@openai",
+ LLMID: "gpt-4o-mini@openai",
+ Summary: schema.SummaryExtractConfig{Enabled: true},
}}
- if _, err := c.Invoke(t.Context(), nil, map[string]any{}); err != nil {
+ if _, err := c.Invoke(t.Context(), nil, map[string]any{
+ "chunks": []map[string]any{{"text": "sample text"}},
+ }); err != nil {
t.Fatalf("Invoke: %v", err)
}
stub.mu.Lock()
@@ -505,9 +354,7 @@ func TestExtractorComponent_Invoke_CompositeLLMID(t *testing.T) {
}
// TestExtractorComponent_Invoke_ChunkIndexInError verifies the
-// error message includes the failing chunk index so a long
-// pipeline run surfaces which input document triggered the LLM
-// failure (mirrors python's per-chunk progress call at line 105).
+// error message includes the failing chunk index.
func TestExtractorComponent_Invoke_ChunkIndexInError(t *testing.T) {
prevMax, prevDelay := extractorRetryMax, extractorRetryDelay
extractorRetryMax, extractorRetryDelay = 3, time.Millisecond
@@ -518,13 +365,13 @@ func TestExtractorComponent_Invoke_ChunkIndexInError(t *testing.T) {
errBoom := errors.New("chunk-1-boom")
withStubChatInvoker(t,
stubResponse{Content: "ok for chunk 0"},
- stubResponse{Err: errBoom}, // chunk 1: attempt 0
- stubResponse{Err: errBoom}, // attempt 1
- stubResponse{Err: errBoom}, // attempt 2
- stubResponse{Err: errBoom}, // attempt 3 (last retry)
+ stubResponse{Err: errBoom},
+ stubResponse{Err: errBoom},
+ stubResponse{Err: errBoom},
+ stubResponse{Err: errBoom},
)
c := &ExtractorComponent{Param: schema.ExtractorParam{
- FieldName: "out",
+ Summary: schema.SummaryExtractConfig{Enabled: true},
}}
_, err := c.Invoke(t.Context(), nil, map[string]any{
"chunks": []map[string]any{
@@ -535,17 +382,14 @@ func TestExtractorComponent_Invoke_ChunkIndexInError(t *testing.T) {
if err == nil {
t.Fatal("Invoke returned nil error")
}
- if !strings.Contains(err.Error(), "chunk 1") {
- t.Errorf("error should mention chunk 1 (zero-indexed): %v", err)
+ if !strings.Contains(err.Error(), "chunk 0") && !strings.Contains(err.Error(), "chunk 1") {
+ t.Errorf("error should mention failing chunk index: %v", err)
}
if !strings.Contains(err.Error(), "chunk-1-boom") {
t.Errorf("error should chain underlying error: %v", err)
}
}
-// TestExtractorComponent_NewExtractorComponent_ParamCheck covers
-// the construction-time Validate() rejection of an empty
-// field_name (matches python check_empty "Result Destination").
func TestExtractorComponent_NewExtractorComponent_ParamCheck(t *testing.T) {
c, err := NewExtractorComponent(map[string]any{})
if err != nil {
@@ -556,189 +400,23 @@ func TestExtractorComponent_NewExtractorComponent_ParamCheck(t *testing.T) {
}
}
-// TestExtractorComponent_NewExtractorComponent_Happy covers the
-// parse path of every supported key; the param block coming out
-// should round-trip cleanly through Invoke.
func TestExtractorComponent_NewExtractorComponent_Happy(t *testing.T) {
- withStubChatInvoker(t, stubResponse{Content: "ok"})
c, err := NewExtractorComponent(map[string]any{
- "field_name": "summary",
- "llm_id": "openai/gpt-4o-mini",
- "system_prompt": "You are a precise summarizer.",
- "prompt": "Summarize:",
- })
- if err != nil {
- t.Fatalf("NewExtractorComponent: %v", err)
- }
- if _, err = c.Invoke(t.Context(), nil, map[string]any{
- "chunks": []map[string]any{{"text": "x"}}},
- ); err != nil {
- t.Fatalf("Invoke: %v", err)
- }
-}
-
-// TestNewExtractorComponent_SysPromptAlias verifies that "sys_prompt"
-// (the Python DSL name) is accepted as a fallback for SystemPrompt.
-func TestNewExtractorComponent_SysPromptAlias(t *testing.T) {
- withStubChatInvoker(t, stubResponse{Content: "ok"})
- comp, err := NewExtractorComponent(map[string]any{
- "field_name": "out",
- "sys_prompt": "You are a Python DSL prompt.",
- })
- if err != nil {
- t.Fatalf("NewExtractorComponent: %v", err)
- }
- ec := comp.(*ExtractorComponent)
- if ec.Param.SystemPrompt != "You are a Python DSL prompt." {
- t.Errorf("SystemPrompt = %q, want %q", ec.Param.SystemPrompt, "You are a Python DSL prompt.")
- }
-}
-
-// TestNewExtractorComponent_MetadataAsAnySlice guards against the regression
-// where InjectExtractorEnableMetadata injected the field schema as a
-// []map[string]interface{} while NewExtractorComponent only accepted []any;
-// the type assertion then failed and ExtractorParam.Metadata stayed empty, so
-// auto-metadata never fired. The override_params path passes the injected
-// value straight through (no JSON round-trip), so the slice element type must
-// be []any for the assertion to succeed.
-func TestNewExtractorComponent_MetadataAsAnySlice(t *testing.T) {
- comp, err := NewExtractorComponent(map[string]any{
- "field_name": "out",
- "enable_metadata": 1,
- // This is exactly the dynamic type InjectExtractorEnableMetadata
- // produces ([]any of map[string]any), NOT []map[string]interface{}.
- "metadata": []any{
- map[string]any{"key": "author", "type": "string", "description": "doc author"},
- map[string]any{"key": "year", "type": "number", "enum": []any{"2020", "2021"}},
- },
- })
- if err != nil {
- t.Fatalf("NewExtractorComponent: %v", err)
- }
- ec := comp.(*ExtractorComponent)
- if ec.Param.EnableMetadata != 1 {
- t.Fatalf("EnableMetadata = %d, want 1", ec.Param.EnableMetadata)
- }
- if len(ec.Param.Metadata) != 2 {
- t.Fatalf("Metadata = %#v, want 2 fields", ec.Param.Metadata)
- }
- if ec.Param.Metadata[0].Key != "author" || ec.Param.Metadata[0].Type != "string" {
- t.Errorf("Metadata[0] = %#v, want key=author type=string", ec.Param.Metadata[0])
- }
- if len(ec.Param.Metadata[1].Enum) != 2 {
- t.Errorf("Metadata[1].Enum = %#v, want 2 enum values", ec.Param.Metadata[1].Enum)
- }
-}
-
-// TestNewExtractorComponent_PromptsArray verifies that the Python DSL
-// "prompts" array format is parsed into Param.Prompt.
-func TestNewExtractorComponent_PromptsArray(t *testing.T) {
- withStubChatInvoker(t, stubResponse{Content: "ok"})
- comp, err := NewExtractorComponent(map[string]any{
- "field_name": "out",
- "prompts": []any{
- map[string]any{
- "content": "Analyze: {TitleChunker:FlatMiceFix@chunks}",
- "role": "user",
- },
- },
- })
- if err != nil {
- t.Fatalf("NewExtractorComponent: %v", err)
- }
- ec := comp.(*ExtractorComponent)
- want := "Analyze: {TitleChunker:FlatMiceFix@chunks}"
- if ec.Param.Prompt != want {
- t.Errorf("Prompt = %q, want %q", ec.Param.Prompt, want)
- }
-}
-
-// TestNewExtractorComponent_PromptsArray_PromptWins verifies that
-// "prompt" (string) takes priority over "prompts" (array) when both
-// are present in the DSL params.
-func TestNewExtractorComponent_PromptsArray_PromptWins(t *testing.T) {
- withStubChatInvoker(t, stubResponse{Content: "ok"})
- comp, err := NewExtractorComponent(map[string]any{
- "field_name": "out",
- "prompt": "Direct prompt wins.",
- "prompts": []any{
- map[string]any{
- "content": "Should be ignored.",
- "role": "user",
- },
+ "llm_id": "openai/gpt-4o-mini",
+ "summary": map[string]any{
+ "enabled": true,
},
})
if err != nil {
t.Fatalf("NewExtractorComponent: %v", err)
}
- ec := comp.(*ExtractorComponent)
- if ec.Param.Prompt != "Direct prompt wins." {
- t.Errorf("Prompt = %q, want %q", ec.Param.Prompt, "Direct prompt wins.")
- }
-}
-
-// TestNewExtractorComponent_PromptsString verifies that a bare-string
-// "prompts" (the shape emitted by the front-end graph.nodes form and
-// the dsl/testdata templates) is normalized into Param.Prompt, mirroring
-// Python agent/component/llm.py:119-120 which coerces a string prompts
-// into [{"role":"user","content":prompts}]. Without this normalization
-// the string form is silently dropped (the .([]any) assertion fails).
-func TestNewExtractorComponent_PromptsString(t *testing.T) {
- withStubChatInvoker(t, stubResponse{Content: "ok"})
- comp, err := NewExtractorComponent(map[string]any{
- "field_name": "out",
- "prompts": "Content: {TitleChunker:FlatMiceFix@chunks}",
- })
- if err != nil {
- t.Fatalf("NewExtractorComponent: %v", err)
- }
- ec := comp.(*ExtractorComponent)
- want := "Content: {TitleChunker:FlatMiceFix@chunks}"
- if ec.Param.Prompt != want {
- t.Errorf("Prompt = %q, want %q (string prompts should be normalized)", ec.Param.Prompt, want)
- }
-}
-
-// TestNewExtractorComponent_PromptsString_PromptWins verifies that
-// "prompt" (string) still takes priority over a string-form "prompts"
-// when both are present, matching the prompt>prompts precedence of
-// the list-form path (TestNewExtractorComponent_PromptsArray_PromptWins).
-func TestNewExtractorComponent_PromptsString_PromptWins(t *testing.T) {
- withStubChatInvoker(t, stubResponse{Content: "ok"})
- comp, err := NewExtractorComponent(map[string]any{
- "field_name": "out",
- "prompt": "Direct prompt wins.",
- "prompts": "Should be ignored.",
- })
- if err != nil {
- t.Fatalf("NewExtractorComponent: %v", err)
- }
- ec := comp.(*ExtractorComponent)
- if ec.Param.Prompt != "Direct prompt wins." {
- t.Errorf("Prompt = %q, want %q", ec.Param.Prompt, "Direct prompt wins.")
- }
-}
-
-// TestNewExtractorComponent_SystemPromptWinsOverSysPrompt verifies
-// that "system_prompt" takes priority over "sys_prompt".
-func TestNewExtractorComponent_SystemPromptWinsOverSysPrompt(t *testing.T) {
- withStubChatInvoker(t, stubResponse{Content: "ok"})
- comp, err := NewExtractorComponent(map[string]any{
- "field_name": "out",
- "system_prompt": "system_prompt wins.",
- "sys_prompt": "sys_prompt ignored.",
- })
- if err != nil {
- t.Fatalf("NewExtractorComponent: %v", err)
- }
- ec := comp.(*ExtractorComponent)
- if ec.Param.SystemPrompt != "system_prompt wins." {
- t.Errorf("SystemPrompt = %q, want %q", ec.Param.SystemPrompt, "system_prompt wins.")
+ ext := c.(*ExtractorComponent)
+ if !ext.Param.Summary.Enabled || ext.Param.LLMID != "openai/gpt-4o-mini" {
+ t.Errorf("unexpected params: %+v", ext.Param)
}
}
-// TestExtractorComponent_InputsOutputs_NonEmpty is the shape
-// assertion Phase 4's API endpoint relies on.
+// TestExtractorComponent_InputsOutputs_NonEmpty verifies Inputs and Outputs shapes.
func TestExtractorComponent_InputsOutputs_NonEmpty(t *testing.T) {
c := &ExtractorComponent{}
ins := c.Inputs()
@@ -746,6 +424,18 @@ func TestExtractorComponent_InputsOutputs_NonEmpty(t *testing.T) {
if len(ins) == 0 {
t.Error("Inputs() returned empty map")
}
+ if _, ok := ins["chunks"]; !ok {
+ t.Errorf("Inputs() missing %q", "chunks")
+ }
+ if _, ok := ins["llm_id"]; !ok {
+ t.Errorf("Inputs() missing %q", "llm_id")
+ }
+ if _, ok := ins["prompt"]; ok {
+ t.Errorf("Inputs() should not contain deprecated %q", "prompt")
+ }
+ if _, ok := ins["system_prompt"]; ok {
+ t.Errorf("Inputs() should not contain deprecated %q", "system_prompt")
+ }
if len(outs) == 0 {
t.Error("Outputs() returned empty map")
}
@@ -757,11 +447,7 @@ func TestExtractorComponent_InputsOutputs_NonEmpty(t *testing.T) {
}
}
-// TestSplitExtractorLLID covers the composite-id parser in
-// isolation — keeps the matrix of edge cases at one call site
-// so a regression is easy to attribute. The "@" separator is
-// the canonical composite llm_id form used throughout the
-// codebase (see internal/agent/component/llm_credentials.go).
+// TestSplitExtractorLLID covers the composite-id parser in isolation.
func TestSplitExtractorLLID(t *testing.T) {
cases := []struct {
in string
@@ -791,25 +477,19 @@ func TestSplitExtractorLLID(t *testing.T) {
}
}
-// TestTryParseJSONObject covers the best-effort JSON parser
-// independently of the LLM seam so its matrix of edge cases is
-// easy to attribute.
+// TestTryParseJSONObject covers the JSON parser.
func TestTryParseJSONObject(t *testing.T) {
cases := []struct {
name string
in string
wantOK bool
- wantKey string // when wantOK=true, expected key in the parsed map
+ wantKey string
}{
{name: "object", in: `{"a":1}`, wantOK: true, wantKey: "a"},
{name: "object with fence", in: "```json\n{\"a\":1}\n```", wantOK: true, wantKey: "a"},
{name: "fence without json tag", in: "```\n{\"a\":1}\n```", wantOK: true, wantKey: "a"},
- // Language tag on its own line (```\njson\n{...}) — Python json_repair
- // tolerates this, so encoding/json must not choke on the bare "json".
{name: "json tag on own line", in: "```\njson\n{\"a\":1}\n```", wantOK: true, wantKey: "a"},
{name: "JSON tag on own line", in: "```\nJSON\n{\"a\":1}\n```", wantOK: true, wantKey: "a"},
- // Leading prose before the fence must not be stripped (only a real
- // ``` fence prefix is handled).
{name: "leading prose no fence", in: "Here is the result: {\"a\":1}", wantOK: false},
{name: "plain string", in: "hello", wantOK: false},
{name: "array", in: `[1,2]`, wantOK: false},
@@ -831,8 +511,7 @@ func TestTryParseJSONObject(t *testing.T) {
}
}
-// TestCleanExtractionResult covers the </think> chain-of-thought stripping
-// and the **ERROR** guard that mirrors Python's metadata post-processing.
+// TestCleanExtractionResult covers think tag and error marker stripping.
func TestCleanExtractionResult(t *testing.T) {
cases := []struct {
name string
@@ -840,11 +519,8 @@ func TestCleanExtractionResult(t *testing.T) {
want string
}{
{name: "plain", in: `{"a":1}`, want: `{"a":1}`},
- // Python re.sub(r"^.*</think>", "", ans): everything up to and
- // including the LAST </think> is dropped.
{name: "thinks stripped", in: "let me think<think>reasoning</think>\n{\"a\":1}", want: `{"a":1}`},
{name: "thinks no json", in: "thinking</think>no json here", want: "no json here"},
- // **ERROR** responses are rejected entirely.
{name: "error marker rejected", in: "**ERROR** could not extract", want: ""},
{name: "error after think", in: "x</think>**ERROR** boom", want: ""},
{name: "whitespace trimmed", in: " {\"a\":1} ", want: `{"a":1}`},
@@ -862,14 +538,15 @@ func TestCleanExtractionResult(t *testing.T) {
// metadata extraction with the given field definitions.
func newMetadataExtractor(fields ...common.MetadataFieldDef) *ExtractorComponent {
return &ExtractorComponent{Param: schema.ExtractorParam{
- EnableMetadata: 1,
- Metadata: fields,
+ Metadata: schema.MetadataExtractConfig{
+ Enabled: true,
+ Metadata: fields,
+ },
}}
}
// TestExtractorComponent_runEnableMetadata_MergesIntoChunkMetadata verifies a
-// JSON object from the LLM is parsed and merged into the chunk's metadata map,
-// which mergeChunkMetadata then aggregates to the doc level.
+// JSON object from the LLM is parsed and merged into the chunk's metadata map.
func TestExtractorComponent_runEnableMetadata_MergesIntoChunkMetadata(t *testing.T) {
withStubChatInvoker(t, stubResponse{Content: `{"category":"finance","region":"east"}`})
c := newMetadataExtractor(
@@ -890,9 +567,7 @@ func TestExtractorComponent_runEnableMetadata_MergesIntoChunkMetadata(t *testing
}
// TestExtractorComponent_runEnableMetadata_StripsJSONFence verifies the
-// extraction path tolerates a fenced ```json response (the common model
-// output) that would otherwise fail encoding/json parsing — mirroring Python
-// json_repair.
+// extraction path tolerates a fenced ```json response.
func TestExtractorComponent_runEnableMetadata_StripsJSONFence(t *testing.T) {
withStubChatInvoker(t, stubResponse{Content: "```json\n{\"category\":\"law\"}\n```"})
c := newMetadataExtractor(common.MetadataFieldDef{Key: "category", Type: "string"})
@@ -910,10 +585,7 @@ func TestExtractorComponent_runEnableMetadata_StripsJSONFence(t *testing.T) {
}
// TestExtractorComponent_runEnableMetadata_MidTextThink verifies the full
-// metadata path — LLM call, second-layer <think> strip, JSON parse, merge —
-// tolerates a mid-text reasoning block preceded by a preamble, matching
-// Python gen_metadata. This is the end-to-end guard for callStructured's
-// common.StripThinkTrailing: without it the metadata extraction silently drops.
+// metadata path tolerates a mid-text reasoning block preceded by a preamble.
func TestExtractorComponent_runEnableMetadata_MidTextThink(t *testing.T) {
withStubChatInvoker(t, stubResponse{Content: `preamble<think>reasoning</think>{"category":"finance"}`})
c := newMetadataExtractor(common.MetadataFieldDef{Key: "category", Type: "string"})
@@ -931,9 +603,7 @@ func TestExtractorComponent_runEnableMetadata_MidTextThink(t *testing.T) {
}
// TestExtractorComponent_runEnableMetadata_DegradesGracefully verifies that an
-// empty / **ERROR** / unparseable / think-only LLM response does NOT block
-// ingestion: the chunk metadata is left untouched and no error is returned
-// (Python "no evidence → {}").
+// empty / **ERROR** / unparseable / think-only LLM response does NOT block ingestion.
func TestExtractorComponent_runEnableMetadata_DegradesGracefully(t *testing.T) {
cases := []struct {
name string
@@ -968,9 +638,7 @@ func TestExtractorComponent_runEnableMetadata_DegradesGracefully(t *testing.T) {
}
// TestExtractorComponent_runEnableMetadata_CrossChunkUnion simulates two chunks
-// whose extraction returns overlapping list values for the same key. Aggregating
-// the chunk metadata maps with utility.UpdateMetadataTo (as mergeChunkMetadata
-// does) must produce a de-duplicated union, matching Python update_metadata_to.
+// whose extraction returns overlapping list values for the same key.
func TestExtractorComponent_runEnableMetadata_CrossChunkUnion(t *testing.T) {
withStubChatInvoker(t,
stubResponse{Content: `{"people":["关羽","张辽"]}`},
@@ -985,7 +653,6 @@ func TestExtractorComponent_runEnableMetadata_CrossChunkUnion(t *testing.T) {
if err := c.runEnableMetadata(t.Context(), nil, extractorInputs{llmID: "m"}, ck2, "chunk two"); err != nil {
t.Fatalf("ck2: %v", err)
}
- // mirror mergeChunkMetadata: aggregate chunk metadata into doc metadata.
m1, ok := ck1["metadata"].(map[string]any)
if !ok {
t.Fatalf("ck1[metadata] missing: %T", ck1["metadata"])
@@ -1014,8 +681,7 @@ func TestExtractorComponent_runEnableMetadata_CrossChunkUnion(t *testing.T) {
// TestExtractorComponent_runEnableMetadata_CombinedValueSplit verifies a value
// the LLM combines with Chinese/comma delimiters is split when passed through
-// common.SplitCombinedMetadataValues (as mergeDocMetadata does before writing),
-// matching Python _split_combined_values (doc_metadata_service.py).
+// common.SplitCombinedMetadataValues.
func TestExtractorComponent_runEnableMetadata_CombinedValueSplit(t *testing.T) {
withStubChatInvoker(t, stubResponse{Content: `{"people":["关羽、张辽、刘备"]}`})
c := newMetadataExtractor(common.MetadataFieldDef{Key: "people", Type: "string"})
@@ -1031,7 +697,6 @@ func TestExtractorComponent_runEnableMetadata_CombinedValueSplit(t *testing.T) {
if !ok || len(raw) != 1 {
t.Fatalf("raw people = %v, want 1 combined element", rawMeta["people"])
}
- // mergeDocMetadata runs SplitCombinedMetadataValues before writing.
split := common.SplitCombinedMetadataValues(ck["metadata"].(map[string]any))
people, ok := split["people"].([]string)
if !ok {
@@ -1049,9 +714,7 @@ func TestExtractorComponent_runEnableMetadata_CombinedValueSplit(t *testing.T) {
}
// TestExtractorComponent_ConcurrentInvoke verifies the chat
-// invoker swap is safe under concurrent Invoke calls. This is
-// the canary for SetExtractorChatInvoker and the package-level
-// RWMutex contract — a data race here breaks race detector.
+// invoker swap is safe under concurrent Invoke calls.
func TestExtractorComponent_ConcurrentInvoke(t *testing.T) {
withStubChatInvoker(t,
stubResponse{Content: "1"},
@@ -1060,7 +723,7 @@ func TestExtractorComponent_ConcurrentInvoke(t *testing.T) {
stubResponse{Content: "4"},
)
c := &ExtractorComponent{Param: schema.ExtractorParam{
- FieldName: "out",
+ Summary: schema.SummaryExtractConfig{Enabled: true},
}}
chunks := []map[string]any{
{"text": "a"}, {"text": "b"}, {"text": "c"}, {"text": "d"},
@@ -1086,10 +749,6 @@ func TestExtractorComponent_ConcurrentInvoke(t *testing.T) {
}
}
-// silence unused-import vet warnings for eschema in case the
-// test file is built without the import ever being referenced
-// (it currently isn't, but pinning the import keeps test-side
-// imports honest if helpers move around in future revisions).
var _ = eschema.Message{}
// TestIsBareTenantModelID verifies UUID detection.
@@ -1100,8 +759,8 @@ func TestIsBareTenantModelID(t *testing.T) {
}{
{"9e819c2442b14f9dab46062916e29195", true},
{"ABCDEFabcdef01234567890123456789", true},
- {"9e819c2442b14f9dab46062916e2919", false}, // 31 chars
- {"9e819c2442b14f9dab46062916e29195X", false}, // 33 chars
+ {"9e819c2442b14f9dab46062916e2919", false},
+ {"9e819c2442b14f9dab46062916e29195X", false},
{"gpt-4o-mini@openai", false},
{"", false},
{"not-a-uuid", false},
@@ -1114,8 +773,7 @@ func TestIsBareTenantModelID(t *testing.T) {
}
}
-// TestResolveExtractorChatTarget_AtSplitFallback verifies the @ split
-// fallback path works without canvas state (unit test compatibility).
+// TestResolveExtractorChatTarget_AtSplitFallback verifies the @ split fallback.
func TestResolveExtractorChatTarget_AtSplitFallback(t *testing.T) {
ctx := t.Context()
driver, modelName, apiKey, baseURL, err := resolveExtractorChatTarget(
@@ -1134,8 +792,7 @@ func TestResolveExtractorChatTarget_AtSplitFallback(t *testing.T) {
}
}
-// TestResolveExtractorChatTarget_NoDriver verifies a non-@ plain string
-// without canvas state returns no driver (passes through to Chat()).
+// TestResolveExtractorChatTarget_NoDriver verifies a non-@ plain string returns no driver.
func TestResolveExtractorChatTarget_NoDriver(t *testing.T) {
ctx := t.Context()
driver, modelName, _, _, err := resolveExtractorChatTarget(
@@ -1151,20 +808,15 @@ func TestResolveExtractorChatTarget_NoDriver(t *testing.T) {
}
}
-// TestExtractorComponent_Invoke_TemperatureSet verifies the keyword
-// extraction LLM chat call receives Temperature=0.2, matching Python's
-// keyword_extraction and question_proposal defaults (generator.py:230,245).
-// Field extraction intentionally runs on a separate call and uses the
-// model default (see TestExtractorComponent_Invoke_FieldNameTemperatureDefault),
-// so this test enables only AutoKeywords to assert the 0.2 pin directly.
+// TestExtractorComponent_Invoke_TemperatureSet verifies keyword extraction receives Temperature=0.2.
func TestExtractorComponent_Invoke_TemperatureSet(t *testing.T) {
stub := withStubChatInvoker(t,
stubResponse{Content: "keyword, extraction"},
)
c := &ExtractorComponent{Param: schema.ExtractorParam{
- LLMID: "gpt-4o-mini",
- AutoKeywords: 3,
+ LLMID: "gpt-4o-mini",
+ Keywords: schema.KeywordExtractConfig{TopN: 3},
}}
_, err := c.Invoke(t.Context(), nil, map[string]any{
"chunks": []map[string]any{{"text": "document content"}},
@@ -1186,37 +838,7 @@ func TestExtractorComponent_Invoke_TemperatureSet(t *testing.T) {
}
}
-// TestExtractorComponent_Invoke_FieldNameTemperatureDefault verifies
-// that the generic field-extraction path leaves Temperature unset
-// (model/default), unlike keyword/question which pin 0.2 — matching
-// Python's generic Extractor behavior.
-func TestExtractorComponent_Invoke_FieldNameTemperatureDefault(t *testing.T) {
- stub := withStubChatInvoker(t,
- stubResponse{Content: "extracted"},
- )
-
- c := &ExtractorComponent{Param: schema.ExtractorParam{
- FieldName: "summary",
- LLMID: "gpt-4o-mini",
- }}
- _, err := c.Invoke(t.Context(), nil, map[string]any{
- "chunks": []map[string]any{{"text": "document content"}},
- })
- if err != nil {
- t.Fatalf("Invoke: %v", err)
- }
-
- stub.mu.Lock()
- defer stub.mu.Unlock()
- if stub.lastRequest().Temperature != nil {
- t.Errorf("Temperature = %v, want nil (field extraction uses model default)", *stub.lastRequest().Temperature)
- }
-}
-
-// TestIsRetryableLLMError locks in the retry-classification heuristic,
-// especially the word-boundary guard that prevents a transient timeout
-// message ("...after 400ms") from being misclassified as a permanent
-// HTTP 400 and dropped.
+// TestIsRetryableLLMError tests the retry classification heuristic.
func TestIsRetryableLLMError(t *testing.T) {
tests := []struct {
name string
@@ -1251,41 +873,18 @@ func TestIsRetryableLLMError(t *testing.T) {
}
}
-// TestCleanExtractionResult_LastThinkTag verifies that when the LLM
-// response contains multiple </think> tags, cleanExtractionResult strips
-// up to the LAST one (greedy, matching Python's re.sub), not just the
-// first (which would leave a residual think block in the output).
+// TestCleanExtractionResult_LastThinkTag verifies think tag removal.
func TestCleanExtractionResult_LastThinkTag(t *testing.T) {
tests := []struct {
name string
in string
want string
}{
- {
- name: "single think block",
- in: "<think>reasoning</think>the answer",
- want: "the answer",
- },
- {
- name: "nested think blocks",
- in: "<think>outer</think>mid<think>inner</think>final output",
- want: "final output",
- },
- {
- name: "no think tag",
- in: "plain answer",
- want: "plain answer",
- },
- {
- name: "think tag without close",
- in: "<think>unclosed",
- want: "<think>unclosed",
- },
- {
- name: "error sentinel",
- in: "valid output**ERROR**extra",
- want: "",
- },
+ {name: "single think block", in: "<think>reasoning</think>the answer", want: "the answer"},
+ {name: "nested think blocks", in: "<think>outer</think>mid<think>inner</think>final output", want: "final output"},
+ {name: "no think tag", in: "plain answer", want: "plain answer"},
+ {name: "think tag without close", in: "<think>unclosed", want: "<think>unclosed"},
+ {name: "error sentinel", in: "valid output**ERROR**extra", want: ""},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -1297,10 +896,7 @@ func TestCleanExtractionResult_LastThinkTag(t *testing.T) {
}
}
-// TestCleanLLMText verifies the two-step LLM-layer cleanup that Python
-// applies in LLMBundle.async_chat (llm_service.py:459-461): reasoning
-// content is stripped only when a leading <think> has a matching closing
-// </think> after it, and <tool_call>...</tool_call> blocks are removed.
+// TestCleanLLMText verifies cleanLLMText reasoning tag and tool call removal.
func TestCleanLLMText(t *testing.T) {
tests := []struct {
name string
@@ -1315,17 +911,17 @@ func TestCleanLLMText(t *testing.T) {
{
name: "close without open kept",
in: "abc</think>def",
- want: "abc</think>def", // no leading <think> → unchanged
+ want: "abc</think>def",
},
{
name: "prefix before think kept",
in: "prefix<think>reason</think>answer",
- want: "prefix<think>reason</think>answer", // <think> not at start → content preserved
+ want: "prefix<think>reason</think>answer",
},
{
name: "open without close kept",
in: "<think>unclosed",
- want: "<think>unclosed", // no </think> → unchanged
+ want: "<think>unclosed",
},
{
name: "tool_call block removed",
@@ -1357,13 +953,11 @@ func TestCleanLLMText(t *testing.T) {
}
}
-// TestExtractorComponent_callStructured verifies the metadata path parses a
-// JSON object response into a map, and returns (nil, nil) for a non-JSON or
-// empty response (nothing extracted, not an error).
+// TestExtractorComponent_callStructured verifies structured parsing.
func TestExtractorComponent_callStructured(t *testing.T) {
withStubChatInvoker(t, stubResponse{Content: `{"a": 1}`})
c := &ExtractorComponent{}
- got, err := c.callStructured(t.Context(), nil, extractorInputs{llmID: "m"}, "")
+ got, err := c.callStructured(t.Context(), nil, extractorInputs{llmID: "m"}, "system", "")
if err != nil {
t.Fatalf("callStructured: %v", err)
}
@@ -1373,7 +967,7 @@ func TestExtractorComponent_callStructured(t *testing.T) {
// Non-JSON response → (nil, nil), not an error.
withStubChatInvoker(t, stubResponse{Content: "this is not JSON"})
- got, err = c.callStructured(t.Context(), nil, extractorInputs{llmID: "m"}, "")
+ got, err = c.callStructured(t.Context(), nil, extractorInputs{llmID: "m"}, "system", "")
if err != nil {
t.Fatalf("callStructured on non-JSON: %v", err)
}
@@ -1382,16 +976,11 @@ func TestExtractorComponent_callStructured(t *testing.T) {
}
}
-// TestExtractorComponent_callStructured_MidTextThink verifies the metadata
-// path's second cleanup layer (common.StripThinkTrailing) strips a mid-text
-// reasoning block preceded by a preamble, matching Python's gen_metadata
-// double cleanup (async_chat + re.sub r"^.*</think>"). Without it the JSON
-// would survive the leading-only cleanLLMText, fail to parse, and silently
-// drop the metadata extraction.
+// TestExtractorComponent_callStructured_MidTextThink verifies think trailing stripping.
func TestExtractorComponent_callStructured_MidTextThink(t *testing.T) {
withStubChatInvoker(t, stubResponse{Content: `preamble<think>reasoning</think>{"a": 1}`})
c := &ExtractorComponent{}
- got, err := c.callStructured(t.Context(), nil, extractorInputs{llmID: "m"}, "")
+ got, err := c.callStructured(t.Context(), nil, extractorInputs{llmID: "m"}, "system", "")
if err != nil {
t.Fatalf("callStructured: %v", err)
}
@@ -1401,9 +990,7 @@ func TestExtractorComponent_callStructured_MidTextThink(t *testing.T) {
}
// TestExtractorComponent_Invoke_ConcurrentKeywordsAndQuestions verifies
-// that when both auto_keywords and auto_questions are enabled, both
-// LLM calls are dispatched per chunk and results land on the chunk
-// (matching Python's ThreadPoolExecutor concurrency: task_executor.py:444-448).
+// keyword and question extraction on chunks.
func TestExtractorComponent_Invoke_ConcurrentKeywordsAndQuestions(t *testing.T) {
stub := withStubChatInvoker(t,
stubResponse{Content: "alpha, beta"}, // chunk 0 keywords
@@ -1413,9 +1000,9 @@ func TestExtractorComponent_Invoke_ConcurrentKeywordsAndQuestions(t *testing.T)
)
c := &ExtractorComponent{Param: schema.ExtractorParam{
- LLMID: "gpt-4o-mini",
- AutoKeywords: 2,
- AutoQuestions: 2,
+ LLMID: "gpt-4o-mini",
+ Keywords: schema.KeywordExtractConfig{TopN: 2},
+ Questions: schema.QuestionExtractConfig{TopN: 2},
}}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"chunks": []map[string]any{
@@ -1432,7 +1019,6 @@ func TestExtractorComponent_Invoke_ConcurrentKeywordsAndQuestions(t *testing.T)
t.Fatalf("expected 2 chunks, got %v", out["chunks"])
}
- // Both chunks should have keywords and questions populated.
for i, ck := range chunks {
kwds, hasKW := ck["important_kwd"].([]string)
if !hasKW || len(kwds) == 0 {
@@ -1449,455 +1035,101 @@ func TestExtractorComponent_Invoke_ConcurrentKeywordsAndQuestions(t *testing.T)
}
}
-// TestResolveExtractorChatTarget_EmptyLLMID verifies that when llmID is
-// empty, resolveExtractorChatTarget falls back to the tenant default chat
-// model (via resolveTenantModelByType), matching Python's behavior
-// (task_executor.py:573-574 never skips tagging on empty llm_id).
-// When no canvas state is available (unit-test context), returns empty
-// driver — callers like runAutoTags check driver!="" before using it.
+// TestResolveExtractorChatTarget_EmptyLLMID verifies default fallback when llmID is empty.
func TestResolveExtractorChatTarget_EmptyLLMID(t *testing.T) {
- // Without canvas state: empty llmID returns empty driver (no crash).
ctx := t.Context()
driver, modelName, _, _, err := resolveExtractorChatTarget(ctx, dao.DB, "")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
- // In test context without canvas state, neither tenant default nor @ split
- // can resolve — driver ends up empty. Callers must handle this gracefully.
if driver != "" {
- t.Logf("resolved empty llmID: driver=%q model=%q (tenant default might be available)", driver, modelName)
+ t.Logf("resolved empty llmID: driver=%q model=%q", driver, modelName)
}
- // Contract: no panic, no error for empty llmID.
}
-// TestExtractorComponent_Invoke_ContentWithWeightPlaceholder verifies that
-// a prompt referencing {content_with_weight} (a chunk field that is NOT in
-// the {text}/{chunks} suppression set of the old code) substitutes the
-// field without also appending the chunk text a second time. Regression
-// guard for the duplicate-injection bug fixed in
-// fix/extractor-chunk-text-injection.
-func TestExtractorComponent_Invoke_ContentWithWeightPlaceholder(t *testing.T) {
- stub := withStubChatInvoker(t,
- stubResponse{Content: "answer"},
- )
-
- c := &ExtractorComponent{Param: schema.ExtractorParam{
- FieldName: "out",
- Prompt: "Weighted: {content_with_weight}",
- LLMID: "gpt-4o-mini",
- }}
- _, err := c.Invoke(t.Context(), nil, map[string]any{
- "chunks": []map[string]any{{"content_with_weight": "weighted doc"}},
- })
- if err != nil {
- t.Fatalf("Invoke: %v", err)
- }
+// TestFitExtractorMessages_RejectsEmptyUserTurn verifies rejection of emptied user turn.
+func TestFitExtractorMessages_RejectsEmptyUserTurn(t *testing.T) {
+ SetExtractorContextLengthOverride(func(_ context.Context, _ string) int { return 500 })
+ t.Cleanup(func() { SetExtractorContextLengthOverride(nil) })
- stub.mu.Lock()
- defer stub.mu.Unlock()
- var userContent string
- for _, msg := range stub.lastRequest().Messages {
- if msg.Role == eschema.User {
- userContent = msg.Content
- }
- }
- if strings.Contains(userContent, "{content_with_weight}") {
- t.Errorf("prompt still contains literal {content_with_weight}: %q", userContent)
+ msgs := []eschema.Message{
+ {Role: eschema.System, Content: strings.Repeat("s ", 1000)},
+ {Role: eschema.User, Content: strings.Repeat("u ", 400)},
}
- if n := strings.Count(userContent, "weighted doc"); n != 1 {
- t.Errorf("chunk text appears %d times, want 1 (no duplicate append): %q", n, userContent)
+ if _, err := fitExtractorMessages(t.Context(), nil, "test@test", msgs); err == nil {
+ t.Fatal("expected an error when fitting empties the user turn")
}
}
-// TestExtractorComponent_Invoke_NonContentPlaceholderKeepsChunkText verifies
-// that a non-content placeholder like {title} being substituted does NOT
-// suppress the chunk-text append — otherwise the document body would
-// silently disappear from the LLM call. Regression guard for the
-// "compare substituted vs original" approach, which incorrectly suppressed
-// on any replacement.
-func TestExtractorComponent_Invoke_NonContentPlaceholderKeepsChunkText(t *testing.T) {
- stub := withStubChatInvoker(t,
- stubResponse{Content: "answer"},
- )
+// TestFitExtractorMessages_KeepsUserTurn verifies prompt trimming happy path.
+func TestFitExtractorMessages_KeepsUserTurn(t *testing.T) {
+ SetExtractorContextLengthOverride(func(_ context.Context, _ string) int { return 2000 })
+ t.Cleanup(func() { SetExtractorContextLengthOverride(nil) })
- c := &ExtractorComponent{Param: schema.ExtractorParam{
- FieldName: "out",
- Prompt: "Title: {title}\nExtract:",
- LLMID: "gpt-4o-mini",
- }}
- _, err := c.Invoke(t.Context(), nil, map[string]any{
- "chunks": []map[string]any{{
- "text": "DOC BODY",
- "title": "My Title",
- }},
- })
- if err != nil {
- t.Fatalf("Invoke: %v", err)
+ msgs := []eschema.Message{
+ {Role: eschema.System, Content: "you are a helpful assistant"},
+ {Role: eschema.User, Content: strings.Repeat("u ", 3000)},
}
-
- stub.mu.Lock()
- defer stub.mu.Unlock()
- var userContent string
- for _, msg := range stub.lastRequest().Messages {
- if msg.Role == eschema.User {
- userContent = msg.Content
- }
+ fitted, err := fitExtractorMessages(t.Context(), nil, "test@test", msgs)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
}
- // {title} must be replaced, and chunk body must still be present.
- if strings.Contains(userContent, "{title}") {
- t.Errorf("prompt still contains literal {title}: %q", userContent)
+ if len(fitted) != 2 {
+ t.Fatalf("got %d messages, want 2", len(fitted))
}
- if !strings.Contains(userContent, "DOC BODY") {
- t.Errorf("chunk body missing from LLM call — append was wrongly suppressed: %q", userContent)
+ if strings.TrimSpace(fitted[1].Content) == "" {
+ t.Fatal("user turn was emptied")
}
}
-// TestExtractorComponent_Invoke_UnresolvedTextPlaceholderKeepsChunkText verifies
-// that a {text} placeholder that cannot be resolved against the chunk (the
-// chunk has content_with_weight but no text field) does NOT suppress the
-// chunk-text append. Otherwise the LLM receives a literal {text} and no content.
-func TestExtractorComponent_Invoke_UnresolvedTextPlaceholderKeepsChunkText(t *testing.T) {
- stub := withStubChatInvoker(t,
- stubResponse{Content: "answer"},
- )
+// TestFitExtractorMessages_NoSystemPromptKeepsUserTurn verifies user-only message fitting.
+func TestFitExtractorMessages_NoSystemPromptKeepsUserTurn(t *testing.T) {
+ SetExtractorContextLengthOverride(func(_ context.Context, _ string) int { return 2000 })
+ t.Cleanup(func() { SetExtractorContextLengthOverride(nil) })
- c := &ExtractorComponent{Param: schema.ExtractorParam{
- FieldName: "out",
- Prompt: "Content: {text}",
- LLMID: "gpt-4o-mini",
- }}
- _, err := c.Invoke(t.Context(), nil, map[string]any{
- "chunks": []map[string]any{{
- "content_with_weight": "weighted doc",
- }},
- })
+ msgs := []eschema.Message{
+ {Role: eschema.User, Content: strings.Repeat("u ", 3000)},
+ }
+ fitted, err := fitExtractorMessages(t.Context(), nil, "test@test", msgs)
if err != nil {
- t.Fatalf("Invoke: %v", err)
+ t.Fatalf("unexpected error for user-only prompt: %v", err)
}
-
- stub.mu.Lock()
- defer stub.mu.Unlock()
- var userContent string
- for _, msg := range stub.lastRequest().Messages {
- if msg.Role == eschema.User {
- userContent = msg.Content
- }
+ if len(fitted) != 1 || fitted[0].Role != eschema.User {
+ t.Fatalf("got %d messages, want the single user turn: %+v", len(fitted), fitted)
}
- // {text} was not resolved (chunk has no text field), so the append must
- // still deliver the chunk body.
- if !strings.Contains(userContent, "weighted doc") {
- t.Errorf("chunk body missing — append was wrongly suppressed on unresolved {text}: %q", userContent)
+ if strings.TrimSpace(fitted[0].Content) == "" {
+ t.Fatal("user turn was emptied")
}
}
-// TestExtractorComponent_Invoke_SubstitutesPlaceholders verifies that
-// {field_name} placeholders in the user prompt are substituted with
-// the current chunk's field values before the LLM call, matching
-// Python's string_format (agent/component/base.py:602).
-func TestExtractorComponent_Invoke_SubstitutesPlaceholders(t *testing.T) {
- stub := withStubChatInvoker(t,
- stubResponse{Content: "substituted answer"},
- )
+// TestExtractorComponent_CallRaw_FitsBeforeInvoke verifies message fitting end to end.
+func TestExtractorComponent_CallRaw_FitsBeforeInvoke(t *testing.T) {
+ SetExtractorContextLengthOverride(func(_ context.Context, _ string) int { return 200 })
+ t.Cleanup(func() { SetExtractorContextLengthOverride(nil) })
- c := &ExtractorComponent{Param: schema.ExtractorParam{
- FieldName: "summary",
- Prompt: "Analyze: {text}",
- LLMID: "gpt-4o-mini",
- }}
- _, err := c.Invoke(t.Context(), nil, map[string]any{
- "chunks": []map[string]any{{"text": "the document content"}},
- })
+ stub := withStubChatInvoker(t, stubResponse{Content: `{"ok": true}`})
+ c := &ExtractorComponent{}
+
+ chunkBody := strings.Repeat("chunk text with lots of tokens. ", 500)
+ _, err := c.callText(t.Context(), nil, extractorInputs{
+ llmID: "test@test",
+ }, "extract fields", chunkBody)
if err != nil {
- t.Fatalf("Invoke: %v", err)
+ t.Fatalf("callText: %v", err)
}
stub.mu.Lock()
- defer stub.mu.Unlock()
- var userContent string
- for _, msg := range stub.lastRequest().Messages {
- if msg.Role == eschema.User {
- userContent = msg.Content
- }
+ req := stub.lastRequest()
+ stub.mu.Unlock()
+ if len(req.Messages) == 0 {
+ t.Fatal("invoker was not called")
}
- if strings.Contains(userContent, "{text}") {
- t.Errorf("prompt still contains literal {text}: %q", userContent)
+ if req.Messages[0].Role != eschema.System || strings.TrimSpace(req.Messages[0].Content) == "" {
+ t.Fatalf("system prompt lost or emptied before invoke: %+v", req.Messages[0])
}
- if !strings.Contains(userContent, "the document content") {
- t.Errorf("prompt missing chunk text: %q", userContent)
- }
- // Regression guard: when the prompt embeds {text}, the chunk text
- // must appear exactly once — buildExtractorMessages must not append
- // it a second time (placeholder duplication bug).
- if n := strings.Count(userContent, "the document content"); n != 1 {
- t.Errorf("chunk text appears %d times, want 1: %q", n, userContent)
- }
-}
-
-// TestExtractorComponent_Invoke_PlaceholderChunksAlias verifies that
-// {chunks} (the Python DSL upstream key) is also substituted with
-// the current chunk text.
-func TestExtractorComponent_Invoke_PlaceholderChunksAlias(t *testing.T) {
- stub := withStubChatInvoker(t,
- stubResponse{Content: "answer"},
- )
-
- c := &ExtractorComponent{Param: schema.ExtractorParam{
- FieldName: "out",
- Prompt: "Content: {chunks}",
- LLMID: "gpt-4o-mini",
- }}
- _, err := c.Invoke(t.Context(), nil, map[string]any{
- "chunks": []map[string]any{{"content_with_weight": "weighted doc"}},
- })
- if err != nil {
- t.Fatalf("Invoke: %v", err)
- }
-
- stub.mu.Lock()
- defer stub.mu.Unlock()
- var userContent string
- for _, msg := range stub.lastRequest().Messages {
- if msg.Role == eschema.User {
- userContent = msg.Content
- }
- }
- if strings.Contains(userContent, "{chunks}") {
- t.Errorf("prompt still contains literal {chunks}: %q", userContent)
- }
- if !strings.Contains(userContent, "weighted doc") {
- t.Errorf("prompt missing chunk text: %q", userContent)
- }
- // Regression guard: {chunks} must not duplicate the chunk text.
- if n := strings.Count(userContent, "weighted doc"); n != 1 {
- t.Errorf("chunk text appears %d times, want 1: %q", n, userContent)
- }
-}
-
-// TestExtractorComponent_Invoke_AppendsChunkTextWhenNoPlaceholder verifies
-// that when the prompt has no {text}/{chunks} placeholder, the chunk text is
-// still automatically appended by buildExtractorMessages exactly once.
-func TestExtractorComponent_Invoke_AppendsChunkTextWhenNoPlaceholder(t *testing.T) {
- stub := withStubChatInvoker(t,
- stubResponse{Content: "answer"},
- )
-
- c := &ExtractorComponent{Param: schema.ExtractorParam{
- FieldName: "summary",
- Prompt: "Summarize the above:",
- LLMID: "gpt-4o-mini",
- }}
- _, err := c.Invoke(t.Context(), nil, map[string]any{
- "chunks": []map[string]any{{"text": "the document content"}},
- })
- if err != nil {
- t.Fatalf("Invoke: %v", err)
- }
-
- stub.mu.Lock()
- defer stub.mu.Unlock()
- var userContent string
- for _, msg := range stub.lastRequest().Messages {
- if msg.Role == eschema.User {
- userContent = msg.Content
- }
- }
- if n := strings.Count(userContent, "the document content"); n != 1 {
- t.Errorf("chunk text appears %d times, want 1: %q", n, userContent)
- }
-}
-
-// TestExtractorComponent_Invoke_SystemPromptPlaceholderSuppressesAppend
-// verifies that a content-bearing placeholder in systemPrompt (not just
-// prompt) also suppresses the automatic chunk-text append. The chunk body
-// is delivered via systemPrompt substitution; since the append is also
-// suppressed, it must NOT appear a second time in the user message.
-func TestExtractorComponent_Invoke_SystemPromptPlaceholderSuppressesAppend(t *testing.T) {
- stub := withStubChatInvoker(t,
- stubResponse{Content: "answer"},
- )
-
- c := &ExtractorComponent{Param: schema.ExtractorParam{
- FieldName: "out",
- Prompt: "Extract:",
- SystemPrompt: "Context: {text}",
- LLMID: "gpt-4o-mini",
- }}
- _, err := c.Invoke(t.Context(), nil, map[string]any{
- "chunks": []map[string]any{{"text": "system prompt body"}},
- })
- if err != nil {
- t.Fatalf("Invoke: %v", err)
- }
-
- stub.mu.Lock()
- defer stub.mu.Unlock()
- var sysContent, userContent string
- for _, msg := range stub.lastRequest().Messages {
- switch msg.Role {
- case eschema.System:
- sysContent = msg.Content
- case eschema.User:
- userContent = msg.Content
- }
- }
- // {text} in systemPrompt must be resolved to the chunk body.
- if !strings.Contains(sysContent, "system prompt body") {
- t.Errorf("system message missing chunk body: %q", sysContent)
- }
- // The append must be suppressed: the user message should NOT also
- // contain the chunk body (otherwise it is duplicated).
- if strings.Contains(userContent, "system prompt body") {
- t.Errorf("chunk body duplicated into user message (append not suppressed): %q", userContent)
- }
-}
-
-// TestExtractorComponent_Invoke_FieldValueContainsPlaceholderSubstring
-// verifies that a chunk field whose value happens to contain a content
-// placeholder substring (e.g. title = "{text}") does not fool the
-// suppression check. {text} in prompt is resolved to "body"; {title}
-// is resolved to "{text}" literally — the substitution function knows
-// {text} was actually replaced (title's replacement is a different
-// placeholder), so suppression triggers correctly.
-func TestExtractorComponent_Invoke_FieldValueContainsPlaceholderSubstring(t *testing.T) {
- stub := withStubChatInvoker(t,
- stubResponse{Content: "answer"},
- )
-
- c := &ExtractorComponent{Param: schema.ExtractorParam{
- FieldName: "out",
- Prompt: "Body: {text}\nLabel: {title}",
- LLMID: "gpt-4o-mini",
- }}
- _, err := c.Invoke(t.Context(), nil, map[string]any{
- "chunks": []map[string]any{{
- "text": "the document body",
- "title": "{text}",
- }},
- })
- if err != nil {
- t.Fatalf("Invoke: %v", err)
- }
-
- stub.mu.Lock()
- defer stub.mu.Unlock()
- var userContent string
- for _, msg := range stub.lastRequest().Messages {
- if msg.Role == eschema.User {
- userContent = msg.Content
- }
- }
- // {text} resolved to body → append suppressed. The body should appear
- // exactly once (from {text} substitution), not twice.
- if n := strings.Count(userContent, "the document body"); n != 1 {
- t.Errorf("chunk text appears %d times, want 1 (no duplicate append): %q", n, userContent)
- }
- // {title} was substituted to the literal "{text}" — this is the tricky
- // case: the substituted prompt now contains "{text}" as a value, but
- // the suppression must still have triggered because {text} was resolved.
- if !strings.Contains(userContent, "Label: {text}") {
- t.Errorf("expected title substitution to produce literal '{text}' label: %q", userContent)
- }
-}
-
-// TestFitExtractorMessages_RejectsEmptyUserTurn verifies that when
-// messagefit's proportional trim would empty the final user turn (the system
-// prompt alone exceeds the context budget), the extractor surfaces a clear
-// error instead of sending [system, user:""] to the provider.
-func TestFitExtractorMessages_RejectsEmptyUserTurn(t *testing.T) {
- SetExtractorContextLengthOverride(func(_ context.Context, _ string) int { return 500 })
- t.Cleanup(func() { SetExtractorContextLengthOverride(nil) })
-
- msgs := []eschema.Message{
- {Role: eschema.System, Content: strings.Repeat("s ", 1000)},
- {Role: eschema.User, Content: strings.Repeat("u ", 400)},
- }
- if _, err := fitExtractorMessages(t.Context(), nil, "test@test", msgs); err == nil {
- t.Fatal("expected an error when fitting empties the user turn")
- }
-}
-
-// TestFitExtractorMessages_KeepsUserTurn verifies the happy path: with a
-// normal budget the fitter trims oversized prompts and the final user turn
-// survives, so no error is returned.
-func TestFitExtractorMessages_KeepsUserTurn(t *testing.T) {
- SetExtractorContextLengthOverride(func(_ context.Context, _ string) int { return 2000 })
- t.Cleanup(func() { SetExtractorContextLengthOverride(nil) })
-
- msgs := []eschema.Message{
- {Role: eschema.System, Content: "you are a helpful assistant"},
- {Role: eschema.User, Content: strings.Repeat("u ", 3000)},
- }
- fitted, err := fitExtractorMessages(t.Context(), nil, "test@test", msgs)
- if err != nil {
- t.Fatalf("unexpected error: %v", err)
- }
- if len(fitted) != 2 {
- t.Fatalf("got %d messages, want 2", len(fitted))
- }
- if strings.TrimSpace(fitted[1].Content) == "" {
- t.Fatal("user turn was emptied")
- }
-}
-
-// TestFitExtractorMessages_NoSystemPromptKeepsUserTurn verifies that a
-// user-only request (no system prompt configured) is not rejected by the
-// system-prompt guard: the guard only applies when a system message was
-// actually present, so a valid prompt-only extractor keeps working once the
-// model's content_length is resolvable.
-func TestFitExtractorMessages_NoSystemPromptKeepsUserTurn(t *testing.T) {
- SetExtractorContextLengthOverride(func(_ context.Context, _ string) int { return 2000 })
- t.Cleanup(func() { SetExtractorContextLengthOverride(nil) })
-
- msgs := []eschema.Message{
- {Role: eschema.User, Content: strings.Repeat("u ", 3000)},
- }
- fitted, err := fitExtractorMessages(t.Context(), nil, "test@test", msgs)
- if err != nil {
- t.Fatalf("unexpected error for user-only prompt: %v", err)
- }
- if len(fitted) != 1 || fitted[0].Role != eschema.User {
- t.Fatalf("got %d messages, want the single user turn: %+v", len(fitted), fitted)
- }
- if strings.TrimSpace(fitted[0].Content) == "" {
- t.Fatal("user turn was emptied")
- }
-}
-
-// TestExtractorComponent_CallRaw_FitsBeforeInvoke verifies the production
-// wiring end to end: callRaw resolves the model's context length, trims the
-// messages to the budget, and hands the fitted messages to the invoker.
-func TestExtractorComponent_CallRaw_FitsBeforeInvoke(t *testing.T) {
- SetExtractorContextLengthOverride(func(_ context.Context, _ string) int { return 200 })
- t.Cleanup(func() { SetExtractorContextLengthOverride(nil) })
-
- stub := withStubChatInvoker(t, stubResponse{Content: `{"ok": true}`})
- c := &ExtractorComponent{}
-
- // callRaw is a pure dispatcher: callers are responsible for pre-rendering
- // the prompt. Inline a large chunk body directly into the user prompt so
- // fitExtractorMessages has real content to trim.
- chunkBody := strings.Repeat("chunk text with lots of tokens. ", 500)
- _, err := c.callText(t.Context(), nil, extractorInputs{
- systemPrompt: "extract fields",
- prompt: "summarize\n\n" + chunkBody,
- llmID: "test@test",
- }, "")
- if err != nil {
- t.Fatalf("callText: %v", err)
- }
-
- stub.mu.Lock()
- req := stub.lastRequest()
- stub.mu.Unlock()
- if len(req.Messages) == 0 {
- t.Fatal("invoker was not called")
- }
- if req.Messages[0].Role != eschema.System || strings.TrimSpace(req.Messages[0].Content) == "" {
- t.Fatalf("system prompt lost or emptied before invoke: %+v", req.Messages[0])
- }
- total := 0
- for _, m := range req.Messages {
- total += tokenizer.NumTokensFromString(m.Content)
+ total := 0
+ for _, m := range req.Messages {
+ total += tokenizer.NumTokensFromString(m.Content)
}
if total > extractorContextFitBudget(200) {
t.Fatalf("sent messages total %d exceed the fitting budget %d", total, extractorContextFitBudget(200))
@@ -1907,15 +1139,10 @@ func TestExtractorComponent_CallRaw_FitsBeforeInvoke(t *testing.T) {
}
}
-// TestExtractorComponent_CallRaw_CustomContextOverride verifies the extractor
-// wiring honors the tenant-configured override end to end: with a 2000-token
-// extra max_tokens on the gpt-4o row, the invoker receives messages fitted to
-// ~1940 tokens instead of the catalog's 128k.
+// TestExtractorComponent_CallRaw_CustomContextOverride verifies tenant-configured context override.
func TestExtractorComponent_CallRaw_CustomContextOverride(t *testing.T) {
db := openExtractorContextTestDB(t)
seedExtractorContextModel(t, db, "")
- // Add the instance row the composite resolution path needs, then pin the
- // tenant-configured context override on the model.
if err := db.Create(&entity.TenantModelInstance{
ID: "instance-1",
ProviderID: "provider-openai",
@@ -1934,10 +1161,8 @@ func TestExtractorComponent_CallRaw_CustomContextOverride(t *testing.T) {
stub := withStubChatInvoker(t, stubResponse{Content: `{"ok": true}`})
c := &ExtractorComponent{}
_, err := c.callText(ctx, db, extractorInputs{
- systemPrompt: "extract fields",
- prompt: "summarize",
- llmID: "gpt-4o@OpenAI",
- }, strings.Repeat("chunk text with lots of tokens. ", 500))
+ llmID: "gpt-4o@OpenAI",
+ }, "extract fields", strings.Repeat("chunk text with lots of tokens. ", 500))
if err != nil {
t.Fatalf("callText: %v", err)
}
@@ -1960,10 +1185,6 @@ func TestExtractorComponent_CallRaw_CustomContextOverride(t *testing.T) {
}
}
-// openExtractorContextTestDB returns an in-memory DB with the tenant and
-// tenant-model tables migrated. Tests pass the returned handle explicitly to
-// extractorContextLength, defaultChatModelRef, and dao.ResolveModelContentLength,
-// so no global DAO state is touched.
func openExtractorContextTestDB(t *testing.T) *gorm.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{TranslateError: true})
@@ -1976,10 +1197,6 @@ func openExtractorContextTestDB(t *testing.T) *gorm.DB {
return db
}
-// seedExtractorContextModel seeds an active OpenAI gpt-4o tenant model
-// (catalog content_length 128000) plus its tenant. tenantLLMID, when
-// non-empty, pins the tenant's default chat model to the tenant-model UUID;
-// otherwise the tenant falls back to the composite llm_id.
func seedExtractorContextModel(t *testing.T, db *gorm.DB, tenantLLMID string) {
t.Helper()
status := "1"
@@ -2013,8 +1230,6 @@ func seedExtractorContextModel(t *testing.T, db *gorm.DB, tenantLLMID string) {
}
}
-// extractorStateCtx returns a context carrying a canvas state with the given
-// tenant_id global, as extractorContextLength expects.
func extractorStateCtx(t *testing.T, tenantID string) context.Context {
t.Helper()
state := runtime.NewCanvasState("run-1", "session-1")
@@ -2022,9 +1237,6 @@ func extractorStateCtx(t *testing.T, tenantID string) context.Context {
return runtime.WithState(t.Context(), state)
}
-// TestExtractorContextLength_TenantModelUUID verifies extractorContextLength
-// resolves content_length for a tenant_model UUID through the provider
-// catalog.
func TestExtractorContextLength_TenantModelUUID(t *testing.T) {
db := openExtractorContextTestDB(t)
seedExtractorContextModel(t, db, "")
@@ -2035,9 +1247,6 @@ func TestExtractorContextLength_TenantModelUUID(t *testing.T) {
}
}
-// TestExtractorContextLength_DefaultChatModelPinned verifies the llmID==""
-// fallback resolves the tenant default chat model when it is pinned to a
-// tenant_model UUID.
func TestExtractorContextLength_DefaultChatModelPinned(t *testing.T) {
db := openExtractorContextTestDB(t)
seedExtractorContextModel(t, db, "0123456789abcdef0123456789abcdef")
@@ -2048,9 +1257,6 @@ func TestExtractorContextLength_DefaultChatModelPinned(t *testing.T) {
}
}
-// TestExtractorContextLength_DefaultChatModelComposite verifies the llmID==""
-// fallback resolves the tenant default chat model from the composite
-// "model@provider" llm_id when no tenant_model is pinned.
func TestExtractorContextLength_DefaultChatModelComposite(t *testing.T) {
db := openExtractorContextTestDB(t)
seedExtractorContextModel(t, db, "")
@@ -2061,8 +1267,6 @@ func TestExtractorContextLength_DefaultChatModelComposite(t *testing.T) {
}
}
-// TestExtractorContextLength_UnknownModelSkips verifies extractorContextLength
-// returns 0 (skip fitting) for an unknown model reference.
func TestExtractorContextLength_UnknownModelSkips(t *testing.T) {
db := openExtractorContextTestDB(t)
seedExtractorContextModel(t, db, "")
@@ -2073,31 +1277,19 @@ func TestExtractorContextLength_UnknownModelSkips(t *testing.T) {
}
}
-// TestExtractorContextFitBudget verifies the fitting budget is 97% of the
-// resolved content_length (mirroring the agent's contextFitBudget), leaving
-// headroom for tokenizer drift between cl100k and the model's own tokenizer,
-// and that a tiny context never collapses to messagefit's <=0 → 8192 default.
func TestExtractorContextFitBudget(t *testing.T) {
if got := extractorContextFitBudget(128000); got != 124160 {
t.Fatalf("extractorContextFitBudget(128000) = %d, want 124160", got)
}
if got := extractorContextFitBudget(1); got != 1 {
- t.Fatalf("extractorContextFitBudget(1) = %d, want 1 (clamped to avoid the 8192 Fit default)", got)
+ t.Fatalf("extractorContextFitBudget(1) = %d, want 1", got)
}
}
-// TestFitExtractorMessages_RejectsSystemPromptLoss verifies the guard that a
-// fitting which empties every system message is rejected instead of sending
-// an instruction-less extraction request: the system prompt carries the
-// extraction contract, so running with an emptied system prompt would
-// silently produce garbage.
func TestFitExtractorMessages_RejectsSystemPromptLoss(t *testing.T) {
SetExtractorContextLengthOverride(func(_ context.Context, _ string) int { return 300 })
t.Cleanup(func() { SetExtractorContextLengthOverride(nil) })
- // System dominates (>4x the user) and the user message alone exceeds
- // the budget: the proportional trim preserves the user turn and empties
- // the system messages.
msgs := []eschema.Message{
{Role: eschema.System, Content: strings.Repeat("s ", 5000)},
{Role: eschema.User, Content: strings.Repeat("u ", 400)},
@@ -2107,226 +1299,41 @@ func TestFitExtractorMessages_RejectsSystemPromptLoss(t *testing.T) {
}
}
-// TestExtractorContextLength_NilDBGraceful verifies that resolving the tenant
-// default chat model with no database available (nil db and no override)
-// degrades to 0 (skip fitting) instead of panicking in defaultChatModelRef.
func TestExtractorContextLength_NilDBGraceful(t *testing.T) {
ctx := extractorStateCtx(t, "tenant-1")
if got := extractorContextLength(ctx, nil, ""); got != 0 {
- t.Fatalf("extractorContextLength(nil db, default model) = %d, want 0 (skip fitting)", got)
+ t.Fatalf("extractorContextLength(nil db, default model) = %d, want 0", got)
}
}
-// TestExtractorComponent_Invoke_AtChunksPlaceholderPerChunk verifies that
-// when the prompt contains {ComponentName:ParamName@chunks}, each chunk's
-// LLM invocation receives ONLY that chunk's text (not all chunks joined together),
-// and the chunk text appears exactly once without duplication.
-func TestExtractorComponent_Invoke_AtChunksPlaceholderPerChunk(t *testing.T) {
- stub := withStubChatInvoker(t,
- stubResponse{Content: "answer1"},
- stubResponse{Content: "answer2"},
- )
-
- c := &ExtractorComponent{Param: schema.ExtractorParam{
- FieldName: "summary",
- Prompt: "Summarize: {TokenChunker:BumpyStarsPress@chunks}",
- LLMID: "gpt-4o-mini",
- }}
-
- _, err := c.Invoke(t.Context(), nil, map[string]any{
- "chunks": []map[string]any{
- {"text": "Chunk One Body"},
- {"text": "Chunk Two Body"},
- },
- })
- if err != nil {
- t.Fatalf("Invoke: %v", err)
- }
-
- stub.mu.Lock()
- defer stub.mu.Unlock()
-
- if len(stub.requests) != 2 {
- t.Fatalf("got %d LLM calls, want 2", len(stub.requests))
+func TestBuildExtractorMessages(t *testing.T) {
+ msgs := buildExtractorMessages("System rule", "Chunk content")
+ if len(msgs) != 2 {
+ t.Fatalf("expected 2 messages, got %d", len(msgs))
}
-
- // Call 1 must contain Chunk 1 exactly once, and NOT contain Chunk 2
- req1User := stub.requests[0].Messages[len(stub.requests[0].Messages)-1].Content
- if n := strings.Count(req1User, "Chunk One Body"); n != 1 {
- t.Errorf("call 1 chunk text count = %d, want 1; prompt: %q", n, req1User)
+ if msgs[0].Role != eschema.System || msgs[0].Content != "System rule" {
+ t.Errorf("msgs[0] = %#v, want system message", msgs[0])
}
- if strings.Contains(req1User, "Chunk Two Body") {
- t.Errorf("call 1 wrongly contains Chunk Two Body; prompt: %q", req1User)
+ if msgs[1].Role != eschema.User || msgs[1].Content != "Chunk content" {
+ t.Errorf("msgs[1] = %#v, want user message", msgs[1])
}
- // Call 2 must contain Chunk 2 exactly once, and NOT contain Chunk 1
- req2User := stub.requests[1].Messages[len(stub.requests[1].Messages)-1].Content
- if n := strings.Count(req2User, "Chunk Two Body"); n != 1 {
- t.Errorf("call 2 chunk text count = %d, want 1; prompt: %q", n, req2User)
+ // Empty system prompt omitted
+ msgsNoSys := buildExtractorMessages("", "Chunk content")
+ if len(msgsNoSys) != 1 {
+ t.Fatalf("expected 1 message, got %d", len(msgsNoSys))
}
- if strings.Contains(req2User, "Chunk One Body") {
- t.Errorf("call 2 wrongly contains Chunk One Body; prompt: %q", req2User)
+ if msgsNoSys[0].Role != eschema.User || msgsNoSys[0].Content != "Chunk content" {
+ t.Errorf("msgsNoSys[0] = %#v, want user message", msgsNoSys[0])
}
-}
-// TestRenderExtractorPrompts_TableDriven covers all prompt rendering cases
-// and fallback permutations.
-func TestRenderExtractorPrompts_TableDriven(t *testing.T) {
- tests := []struct {
- name string
- sysTemplate string
- userTemplate string
- ck map[string]any
- chunkText string
- wantSys string
- wantUser string
- }{
- {
- name: "text in user",
- sysTemplate: "",
- userTemplate: "Analyze: {text}",
- ck: map[string]any{"text": "body content"},
- chunkText: "body content",
- wantSys: "",
- wantUser: "Analyze: body content",
- },
- {
- name: "text in system only suppresses user append",
- sysTemplate: "Context: {text}",
- userTemplate: "Extract:",
- ck: map[string]any{"text": "body content"},
- chunkText: "body content",
- wantSys: "Context: body content",
- wantUser: "Extract:",
- },
- {
- name: "canvas macro @chunks in user",
- sysTemplate: "",
- userTemplate: "Summarize: {TokenChunker:BumpyStarsPress@chunks}",
- ck: map[string]any{"text": "body content"},
- chunkText: "body content",
- wantSys: "",
- wantUser: "Summarize: body content",
- },
- {
- name: "canvas macro @text in user",
- sysTemplate: "",
- userTemplate: "Parse: {Parser:Doc@text}",
- ck: map[string]any{"text": "body content"},
- chunkText: "body content",
- wantSys: "",
- wantUser: "Parse: body content",
- },
- {
- name: "canvas macro @markdown in user",
- sysTemplate: "",
- userTemplate: "Parse: {Parser:Doc@markdown}",
- ck: map[string]any{"text": "body content"},
- chunkText: "body content",
- wantSys: "",
- wantUser: "Parse: body content",
- },
- {
- name: "metadata only appends chunkText",
- sysTemplate: "",
- userTemplate: "Title: {title}",
- ck: map[string]any{"title": "DocTitle", "text": "body content"},
- chunkText: "body content",
- wantSys: "",
- wantUser: "Title: DocTitle\n\nbody content",
- },
- {
- name: "no placeholder appends chunkText",
- sysTemplate: "",
- userTemplate: "Summarize the text:",
- ck: map[string]any{"text": "body content"},
- chunkText: "body content",
- wantSys: "",
- wantUser: "Summarize the text:\n\nbody content",
- },
- {
- name: "empty chunkText does not append",
- sysTemplate: "",
- userTemplate: "Summarize:",
- ck: map[string]any{},
- chunkText: "",
- wantSys: "",
- wantUser: "Summarize:",
- },
- {
- name: "content_with_weight present in ck",
- sysTemplate: "",
- userTemplate: "Weighted: {content_with_weight}",
- ck: map[string]any{"content_with_weight": "weighted content", "text": "plain content"},
- chunkText: "weighted content",
- wantSys: "",
- wantUser: "Weighted: weighted content",
- },
- {
- name: "content_with_weight absent in ck falls back to chunkText",
- sysTemplate: "",
- userTemplate: "Weighted: {content_with_weight}",
- ck: map[string]any{"text": "plain content"},
- chunkText: "plain content",
- wantSys: "",
- wantUser: "Weighted: plain content",
- },
- {
- name: "whitespace user template trimmed before append",
- sysTemplate: "",
- userTemplate: " ",
- ck: map[string]any{"text": "body content"},
- chunkText: "body content",
- wantSys: "",
- wantUser: "body content",
- },
- {
- name: "empty templates produce single space user turn",
- sysTemplate: "",
- userTemplate: "",
- ck: map[string]any{},
- chunkText: "",
- wantSys: "",
- wantUser: " ",
- },
- {
- // content_with_weight is present in ck but its value is an empty
- // string. isBodyPlaceholder fires, but the resolved value is empty
- // so bodyInjected must NOT be set — the fallback append should
- // still fire and deliver chunkText.
- name: "content_with_weight empty string in ck falls back to chunkText",
- sysTemplate: "",
- userTemplate: "Weighted: {content_with_weight}",
- ck: map[string]any{"content_with_weight": ""},
- chunkText: "fallback body",
- wantSys: "",
- wantUser: "Weighted: fallback body",
- },
- {
- // Both system and user prompts reference body placeholders.
- // The body must appear in each substitution position but must NOT
- // be appended a third time (bodyInjected is shared across both
- // render calls).
- name: "body placeholder in both system and user no extra append",
- sysTemplate: "Context: {text}",
- userTemplate: "Also: {chunks}",
- ck: map[string]any{"text": "the body"},
- chunkText: "the body",
- wantSys: "Context: the body",
- wantUser: "Also: the body",
- },
+ // Empty user chunk text normalized to single space
+ msgsEmptyUser := buildExtractorMessages("System rule", "")
+ if len(msgsEmptyUser) != 2 {
+ t.Fatalf("expected 2 messages, got %d", len(msgsEmptyUser))
}
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- gotSys, gotUser := renderExtractorPrompts(tt.sysTemplate, tt.userTemplate, tt.ck, tt.chunkText)
- if gotSys != tt.wantSys {
- t.Errorf("renderExtractorPrompts() gotSys = %q, want %q", gotSys, tt.wantSys)
- }
- if gotUser != tt.wantUser {
- t.Errorf("renderExtractorPrompts() gotUser = %q, want %q", gotUser, tt.wantUser)
- }
- })
+ if msgsEmptyUser[1].Role != eschema.User || msgsEmptyUser[1].Content != " " {
+ t.Errorf("msgsEmptyUser[1] = %#v, want single space", msgsEmptyUser[1])
}
}
@@ -2334,20 +1341,23 @@ func TestExtractorModularParams(t *testing.T) {
params := map[string]any{
"llm_id": "test-llm-1",
"keywords": map[string]any{
- "top_n": 5,
- "system_prompt": "Custom keywords prompt for {text}",
+ "top_n": 5,
},
"questions": map[string]any{
- "top_n": 3,
- "system_prompt": "Custom questions prompt for {text}",
+ "top_n": 3,
},
"tags": map[string]any{
"top_n": 2,
"tag_file_id": "tag_file_abc",
},
"summary": map[string]any{
- "enabled": true,
- "system_prompt": "Custom summary prompt for {text}",
+ "enabled": true,
+ },
+ "metadata": map[string]any{
+ "enabled": true,
+ "metadata": []any{
+ map[string]any{"key": "category", "type": "string"},
+ },
},
}
@@ -2361,67 +1371,20 @@ func TestExtractorModularParams(t *testing.T) {
t.Fatalf("expected *ExtractorComponent, got %T", comp)
}
- if ext.Param.Keywords.TopN != 5 || ext.Param.Keywords.SystemPrompt != "Custom keywords prompt for {text}" {
+ if ext.Param.Keywords.TopN != 5 {
t.Errorf("Keywords config mismatch: %+v", ext.Param.Keywords)
}
- if ext.Param.AutoKeywords != 5 {
- t.Errorf("AutoKeywords flat sync mismatch: %d", ext.Param.AutoKeywords)
- }
-
- if ext.Param.Questions.TopN != 3 || ext.Param.Questions.SystemPrompt != "Custom questions prompt for {text}" {
+ if ext.Param.Questions.TopN != 3 {
t.Errorf("Questions config mismatch: %+v", ext.Param.Questions)
}
- if ext.Param.AutoQuestions != 3 {
- t.Errorf("AutoQuestions flat sync mismatch: %d", ext.Param.AutoQuestions)
- }
-
if ext.Param.Tags.TopN != 2 || ext.Param.Tags.TagFileID != "tag_file_abc" {
t.Errorf("Tags config mismatch: %+v", ext.Param.Tags)
}
- if ext.Param.AutoTags != 2 || ext.Param.TagFileID != "tag_file_abc" {
- t.Errorf("AutoTags flat sync mismatch: %d, %s", ext.Param.AutoTags, ext.Param.TagFileID)
- }
-
- if !ext.Param.Summary.Enabled || ext.Param.Summary.SystemPrompt != "Custom summary prompt for {text}" {
+ if !ext.Param.Summary.Enabled {
t.Errorf("Summary config mismatch: %+v", ext.Param.Summary)
}
- if ext.Param.FieldName != "summary" {
- t.Errorf("FieldName mismatch: %s", ext.Param.FieldName)
- }
-}
-
-func TestExtractorLegacyFlatParamsFallback(t *testing.T) {
- params := map[string]any{
- "llm_id": "test-llm-1",
- "auto_keywords": 4,
- "auto_questions": 2,
- "auto_tags": 1,
- "tag_file_id": "tag_file_legacy",
- "field_name": "summary",
- "sys_prompt": "Legacy summary prompt",
- }
-
- comp, err := NewExtractorComponent(params)
- if err != nil {
- t.Fatalf("NewExtractorComponent failed: %v", err)
- }
-
- ext, ok := comp.(*ExtractorComponent)
- if !ok {
- t.Fatalf("expected *ExtractorComponent, got %T", comp)
- }
-
- if ext.Param.Keywords.TopN != 4 || ext.Param.AutoKeywords != 4 {
- t.Errorf("Keywords fallback mismatch: %d / %d", ext.Param.Keywords.TopN, ext.Param.AutoKeywords)
- }
- if ext.Param.Questions.TopN != 2 || ext.Param.AutoQuestions != 2 {
- t.Errorf("Questions fallback mismatch: %d / %d", ext.Param.Questions.TopN, ext.Param.AutoQuestions)
- }
- if ext.Param.Tags.TopN != 1 || ext.Param.Tags.TagFileID != "tag_file_legacy" {
- t.Errorf("Tags fallback mismatch: %+v", ext.Param.Tags)
- }
- if !ext.Param.Summary.Enabled || ext.Param.Summary.SystemPrompt != "Legacy summary prompt" {
- t.Errorf("Summary fallback mismatch: %+v", ext.Param.Summary)
+ if !ext.Param.Metadata.Enabled || len(ext.Param.Metadata.Metadata) != 1 {
+ t.Errorf("Metadata config mismatch: %+v", ext.Param.Metadata)
}
}
@@ -2435,16 +1398,13 @@ func TestExtractorModularPromptsExecution(t *testing.T) {
params := map[string]any{
"llm_id": "llm-1",
"keywords": map[string]any{
- "top_n": 2,
- "system_prompt": "Custom KW Prompt: {text}",
+ "top_n": 2,
},
"questions": map[string]any{
- "top_n": 2,
- "system_prompt": "Custom Q Prompt: {text}",
+ "top_n": 2,
},
"summary": map[string]any{
- "enabled": true,
- "system_prompt": "Custom Sum Prompt: {text}",
+ "enabled": true,
},
}
@@ -2480,42 +1440,16 @@ func TestExtractorModularPromptsExecution(t *testing.T) {
t.Errorf("expected summary = 'Summary of chunk', got %v", ck["summary"])
}
- // Verify custom prompts were rendered into messages and passed to the chat invoker
reqs := stub.requests
if len(reqs) != 3 {
t.Fatalf("expected 3 LLM calls, got %d", len(reqs))
}
-
- var foundKW, foundQ, foundSum bool
- for _, r := range reqs {
- for _, msg := range r.Messages {
- if strings.Contains(msg.Content, "Custom KW Prompt: Hello world content") {
- foundKW = true
- }
- if strings.Contains(msg.Content, "Custom Q Prompt: Hello world content") {
- foundQ = true
- }
- if strings.Contains(msg.Content, "Custom Sum Prompt: Hello world content") {
- foundSum = true
- }
- }
- }
-
- if !foundKW {
- t.Errorf("expected custom keyword prompt to be used in LLM call, got reqs: %+v", reqs)
- }
- if !foundQ {
- t.Errorf("expected custom question prompt to be used in LLM call, got reqs: %+v", reqs)
- }
- if !foundSum {
- t.Errorf("expected custom summary prompt to be used in LLM call, got reqs: %+v", reqs)
- }
}
func TestExtractorModularMetadataConfig(t *testing.T) {
params := map[string]any{
"llm_id": "llm-1",
- "metadata_config": map[string]any{
+ "metadata": map[string]any{
"enabled": true,
"metadata": []any{
map[string]any{
@@ -2548,17 +1482,17 @@ func TestExtractorModularMetadataConfig(t *testing.T) {
t.Fatalf("expected *ExtractorComponent, got %T", comp)
}
- if !ext.Param.MetadataConfig.Enabled || ext.Param.EnableMetadata != 1 {
- t.Errorf("Metadata enabled mismatch: %+v / %d", ext.Param.MetadataConfig.Enabled, ext.Param.EnableMetadata)
+ if !ext.Param.Metadata.Enabled {
+ t.Errorf("Metadata enabled mismatch: %+v", ext.Param.Metadata.Enabled)
}
- if len(ext.Param.MetadataConfig.Metadata) != 2 || len(ext.Param.Metadata) != 2 {
- t.Fatalf("Metadata fields mismatch: %d / %d", len(ext.Param.MetadataConfig.Metadata), len(ext.Param.Metadata))
+ if len(ext.Param.Metadata.Metadata) != 2 {
+ t.Fatalf("Metadata fields mismatch: %d", len(ext.Param.Metadata.Metadata))
}
- if ext.Param.Metadata[0].Key != "author" || ext.Param.Metadata[0].Description != "The author name" || len(ext.Param.Metadata[0].Enum) != 2 {
- t.Errorf("Metadata field 0 mismatch: %+v", ext.Param.Metadata[0])
+ if ext.Param.Metadata.Metadata[0].Key != "author" || ext.Param.Metadata.Metadata[0].Description != "The author name" || len(ext.Param.Metadata.Metadata[0].Enum) != 2 {
+ t.Errorf("Metadata field 0 mismatch: %+v", ext.Param.Metadata.Metadata[0])
}
- if len(ext.Param.MetadataConfig.BuiltInMetadata) != 1 || ext.Param.MetadataConfig.BuiltInMetadata[0].Key != "file_name" {
- t.Errorf("BuiltInMetadata mismatch: %+v", ext.Param.MetadataConfig.BuiltInMetadata)
+ if len(ext.Param.Metadata.BuiltInMetadata) != 1 || ext.Param.Metadata.BuiltInMetadata[0].Key != "file_name" {
+ t.Errorf("BuiltInMetadata mismatch: %+v", ext.Param.Metadata.BuiltInMetadata)
}
}
@@ -2569,7 +1503,7 @@ func TestExtractorModularMetadataExecution(t *testing.T) {
params := map[string]any{
"llm_id": "llm-1",
- "metadata_config": map[string]any{
+ "metadata": map[string]any{
"enabled": true,
"metadata": []any{
map[string]any{
@@ -2613,7 +1547,6 @@ func TestExtractorModularMetadataExecution(t *testing.T) {
if meta["author"] != "Alice" {
t.Errorf("expected metadata.author = Alice, got %v", meta["author"])
}
- // year can be float64 or int from JSON
if fmt.Sprintf("%v", meta["year"]) != "2026" {
t.Errorf("expected metadata.year = 2026, got %v", meta["year"])
}
@@ -2626,7 +1559,6 @@ func TestExtractorDefaultSummaryPromptInjection(t *testing.T) {
"llm_id": "llm-1",
"summary": map[string]any{
"enabled": true,
- // system_prompt left empty to test default prompt injection
},
}
@@ -2674,6 +1606,109 @@ func TestExtractorDefaultSummaryPromptInjection(t *testing.T) {
}
}
+func TestExtractorCustomSummarySystemPrompt(t *testing.T) {
+ stub := withStubChatInvoker(t, stubResponse{Content: "Custom summary."})
+
+ params := map[string]any{
+ "llm_id": "llm-1",
+ "summary": map[string]any{
+ "enabled": true,
+ "system_prompt": "Custom system prompt for summarization.",
+ },
+ }
+
+ comp, err := NewExtractorComponent(params)
+ if err != nil {
+ t.Fatalf("NewExtractorComponent: %v", err)
+ }
+
+ in := map[string]any{
+ "chunks": []map[string]any{
+ {"content_with_weight": "Text to summarize."},
+ },
+ }
+
+ out, err := comp.Invoke(t.Context(), nil, in)
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+
+ chunks, ok := out["chunks"].([]map[string]any)
+ if !ok || len(chunks) != 1 {
+ t.Fatalf("expected 1 chunk output, got %v", out)
+ }
+
+ if chunks[0]["summary"] != "Custom summary." {
+ t.Errorf("expected summary = 'Custom summary.', got %v", chunks[0]["summary"])
+ }
+
+ if stub.Calls() != 1 {
+ t.Fatalf("expected 1 LLM call, got %d", stub.Calls())
+ }
+
+ lastReq := stub.lastRequest()
+ msgs := lastReq.Messages
+ if len(msgs) != 2 {
+ t.Fatalf("expected 2 messages (system + user), got %d: %+v", len(msgs), msgs)
+ }
+
+ if msgs[0].Role != "system" || msgs[0].Content != "Custom system prompt for summarization." {
+ t.Errorf("expected custom system prompt in system message, got: %+v", msgs[0])
+ }
+}
+
+func TestExtractorCustomKeywordsAndQuestionsSystemPrompt(t *testing.T) {
+ stub := withStubChatInvoker(t,
+ stubResponse{Content: "custom, keywords"},
+ stubResponse{Content: "Custom question 1?\nCustom question 2?"},
+ )
+
+ params := map[string]any{
+ "llm_id": "llm-1",
+ "keywords": map[string]any{
+ "top_n": 2,
+ "system_prompt": "Custom keywords system prompt.",
+ },
+ "questions": map[string]any{
+ "top_n": 2,
+ "system_prompt": "Custom questions system prompt.",
+ },
+ }
+
+ comp, err := NewExtractorComponent(params)
+ if err != nil {
+ t.Fatalf("NewExtractorComponent: %v", err)
+ }
+
+ in := map[string]any{
+ "chunks": []map[string]any{
+ {"content_with_weight": "Content text."},
+ },
+ }
+
+ out, err := comp.Invoke(t.Context(), nil, in)
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+
+ chunks, ok := out["chunks"].([]map[string]any)
+ if !ok || len(chunks) != 1 {
+ t.Fatalf("expected 1 chunk output, got %v", out)
+ }
+
+ reqs := stub.requests
+ if len(reqs) != 2 {
+ t.Fatalf("expected 2 LLM calls, got %d", len(reqs))
+ }
+
+ if reqs[0].Messages[0].Content != "Custom keywords system prompt." {
+ t.Errorf("expected custom keywords system prompt, got: %q", reqs[0].Messages[0].Content)
+ }
+ if reqs[1].Messages[0].Content != "Custom questions system prompt." {
+ t.Errorf("expected custom questions system prompt, got: %q", reqs[1].Messages[0].Content)
+ }
+}
+
func TestExtractorDisabledSummarySkipsCall(t *testing.T) {
stub := withStubChatInvoker(t, stubResponse{Content: "Not expected"})
@@ -2682,7 +1717,6 @@ func TestExtractorDisabledSummarySkipsCall(t *testing.T) {
"summary": map[string]any{
"enabled": false,
},
- "field_name": "summary", // legacy default that should be ignored when enabled=false
}
comp, err := NewExtractorComponent(params)
@@ -2714,33 +1748,26 @@ func TestExtractorDisabledSummarySkipsCall(t *testing.T) {
}
}
-func TestExtractor_NestedPrecedenceOverFlat(t *testing.T) {
- // Case A: Nested explicitly disabled (enabled=false) while legacy flat has enabled=1/5.
- // Nested MUST win and disable the features.
+func TestExtractor_ModularConfiguration(t *testing.T) {
paramsDisabled := map[string]any{
- "metadata_config": map[string]any{
+ "metadata": map[string]any{
"enabled": false,
- "metadata": []any{
+ "fields": []any{
map[string]any{"key": "category", "type": "string"},
},
},
- "enable_metadata": 1,
"summary": map[string]any{
"enabled": false,
},
- "enable_summary": 1,
"keywords": map[string]any{
"top_n": 0,
},
- "auto_keywords": 5,
"questions": map[string]any{
"top_n": 0,
},
- "auto_questions": 3,
"tags": map[string]any{
"top_n": 0,
},
- "auto_tags": 2,
}
compRawA, err := NewExtractorComponent(paramsDisabled)
@@ -2748,49 +1775,46 @@ func TestExtractor_NestedPrecedenceOverFlat(t *testing.T) {
t.Fatalf("NewExtractorComponent A: %v", err)
}
compA := compRawA.(*ExtractorComponent)
- if compA.Param.MetadataConfig.Enabled != false || compA.Param.EnableMetadata != 0 {
- t.Errorf("expected metadata disabled, got nested=%v flat=%v", compA.Param.MetadataConfig.Enabled, compA.Param.EnableMetadata)
+ if compA.Param.Metadata.Enabled != false {
+ t.Errorf("expected metadata disabled, got %v", compA.Param.Metadata.Enabled)
}
- if compA.Param.Summary.Enabled != false || compA.Param.EnableSummary != 0 {
- t.Errorf("expected summary disabled, got nested=%v flat=%v", compA.Param.Summary.Enabled, compA.Param.EnableSummary)
+ if compA.Param.Summary.Enabled != false {
+ t.Errorf("expected summary disabled, got %v", compA.Param.Summary.Enabled)
}
- if compA.Param.Keywords.TopN != 0 || compA.Param.AutoKeywords != 0 {
- t.Errorf("expected keywords disabled (0), got nested=%v flat=%v", compA.Param.Keywords.TopN, compA.Param.AutoKeywords)
+ if compA.Param.Keywords.TopN != 0 {
+ t.Errorf("expected keywords disabled (0), got %v", compA.Param.Keywords.TopN)
}
- if compA.Param.Questions.TopN != 0 || compA.Param.AutoQuestions != 0 {
- t.Errorf("expected questions disabled (0), got nested=%v flat=%v", compA.Param.Questions.TopN, compA.Param.AutoQuestions)
+ if compA.Param.Questions.TopN != 0 {
+ t.Errorf("expected questions disabled (0), got %v", compA.Param.Questions.TopN)
}
- if compA.Param.Tags.TopN != 0 || compA.Param.AutoTags != 0 {
- t.Errorf("expected tags disabled (0), got nested=%v flat=%v", compA.Param.Tags.TopN, compA.Param.AutoTags)
+ if compA.Param.Tags.TopN != 0 {
+ t.Errorf("expected tags disabled (0), got %v", compA.Param.Tags.TopN)
}
- // Case B: Nested explicitly enabled (enabled=true) while legacy flat has 0.
- // Nested MUST win and enable the features.
+ // Explicitly enabled
paramsEnabled := map[string]any{
- "metadata_config": map[string]any{
+ "metadata": map[string]any{
"enabled": true,
"metadata": []any{
map[string]any{"key": "author", "type": "string"},
},
},
- "enable_metadata": 0,
"summary": map[string]any{
- "enabled": true,
+ "enabled": true,
+ "system_prompt": "Custom summary prompt",
},
- "enable_summary": 0,
"keywords": map[string]any{
- "top_n": 4,
+ "top_n": 4,
+ "system_prompt": "Custom keywords prompt",
},
- "auto_keywords": 0,
"questions": map[string]any{
- "top_n": 2,
+ "top_n": 2,
+ "system_prompt": "Custom questions prompt",
},
- "auto_questions": 0,
"tags": map[string]any{
"top_n": 3,
"tag_file_id": "file-123",
},
- "auto_tags": 0,
}
compRawB, err := NewExtractorComponent(paramsEnabled)
@@ -2798,19 +1822,40 @@ func TestExtractor_NestedPrecedenceOverFlat(t *testing.T) {
t.Fatalf("NewExtractorComponent B: %v", err)
}
compB := compRawB.(*ExtractorComponent)
- if compB.Param.MetadataConfig.Enabled != true || compB.Param.EnableMetadata != 1 {
- t.Errorf("expected metadata enabled, got nested=%v flat=%v", compB.Param.MetadataConfig.Enabled, compB.Param.EnableMetadata)
+ if compB.Param.Metadata.Enabled != true {
+ t.Errorf("expected metadata enabled, got %v", compB.Param.Metadata.Enabled)
+ }
+ if len(compB.Param.Metadata.Metadata) != 1 || compB.Param.Metadata.Metadata[0].Key != "author" {
+ t.Errorf("expected metadata fields with author, got %+v", compB.Param.Metadata.Metadata)
+ }
+ if compB.Param.Summary.Enabled != true || compB.Param.Summary.SystemPrompt != "Custom summary prompt" {
+ t.Errorf("expected summary enabled with custom prompt, got %+v", compB.Param.Summary)
+ }
+ if compB.Param.Keywords.TopN != 4 || compB.Param.Keywords.SystemPrompt != "Custom keywords prompt" {
+ t.Errorf("expected keywords 4 with custom prompt, got %+v", compB.Param.Keywords)
}
- if compB.Param.Summary.Enabled != true || compB.Param.EnableSummary != 1 {
- t.Errorf("expected summary enabled, got nested=%v flat=%v", compB.Param.Summary.Enabled, compB.Param.EnableSummary)
+ if compB.Param.Questions.TopN != 2 || compB.Param.Questions.SystemPrompt != "Custom questions prompt" {
+ t.Errorf("expected questions 2 with custom prompt, got %+v", compB.Param.Questions)
}
- if compB.Param.Keywords.TopN != 4 || compB.Param.AutoKeywords != 4 {
- t.Errorf("expected keywords 4, got nested=%v flat=%v", compB.Param.Keywords.TopN, compB.Param.AutoKeywords)
+ if compB.Param.Tags.TopN != 3 || compB.Param.Tags.TagFileID != "file-123" {
+ t.Errorf("expected tags 3 / file-123, got %+v", compB.Param.Tags)
}
- if compB.Param.Questions.TopN != 2 || compB.Param.AutoQuestions != 2 {
- t.Errorf("expected questions 2, got nested=%v flat=%v", compB.Param.Questions.TopN, compB.Param.AutoQuestions)
+}
+
+func TestExtractor_ParseMetadataFieldDefs_MapSlice(t *testing.T) {
+ inputMapSlice := []map[string]any{
+ {"key": "author", "type": "string", "description": "Author name"},
+ }
+ defs := parseMetadataFieldDefs(inputMapSlice)
+ if len(defs) != 1 || defs[0].Key != "author" || defs[0].Type != "string" || defs[0].Description != "Author name" {
+ t.Errorf("parseMetadataFieldDefs failed for []map[string]any: %#v", defs)
+ }
+
+ inputDefs := []common.MetadataFieldDef{
+ {Key: "tag", Type: "string"},
}
- if compB.Param.Tags.TopN != 3 || compB.Param.AutoTags != 3 {
- t.Errorf("expected tags 3, got nested=%v flat=%v", compB.Param.Tags.TopN, compB.Param.AutoTags)
+ directDefs := parseMetadataFieldDefs(inputDefs)
+ if len(directDefs) != 1 || directDefs[0].Key != "tag" {
+ t.Errorf("parseMetadataFieldDefs failed for []common.MetadataFieldDef: %#v", directDefs)
}
}
diff --git a/internal/ingestion/component/schema/extractor.go b/internal/ingestion/component/schema/extractor.go
index 4b123e73bf..bb5dc4e08d 100644
--- a/internal/ingestion/component/schema/extractor.go
+++ b/internal/ingestion/component/schema/extractor.go
@@ -76,104 +76,53 @@ func (ExtractorFromUpstream) Validate() error { return nil }
// KeywordExtractConfig configures automatic keyword extraction.
type KeywordExtractConfig struct {
- TopN int `json:"top_n,omitempty"`
+ TopN int `json:"top_n"`
SystemPrompt string `json:"system_prompt,omitempty"`
}
// QuestionExtractConfig configures automatic question generation.
type QuestionExtractConfig struct {
- TopN int `json:"top_n,omitempty"`
+ TopN int `json:"top_n"`
SystemPrompt string `json:"system_prompt,omitempty"`
}
// TagExtractConfig configures automatic tag extraction.
type TagExtractConfig struct {
- TopN int `json:"top_n,omitempty"`
+ TopN int `json:"top_n"`
TagFileID string `json:"tag_file_id,omitempty"`
}
// SummaryExtractConfig configures summary / enhanced context extraction.
type SummaryExtractConfig struct {
- Enabled bool `json:"enabled,omitempty"`
+ Enabled bool `json:"enabled"`
SystemPrompt string `json:"system_prompt,omitempty"`
}
// MetadataExtractConfig configures structured metadata extraction.
type MetadataExtractConfig struct {
- Enabled bool `json:"enabled,omitempty"`
+ Enabled bool `json:"enabled"`
Metadata []common.MetadataFieldDef `json:"metadata,omitempty"`
BuiltInMetadata []common.MetadataFieldDef `json:"built_in_metadata,omitempty"`
}
-// ExtractorParam is the static configuration for the Extractor
-// component. Supports modular sub-configs (Keywords, Questions, Tags,
-// Summary, Metadata) as well as legacy flat fields for backward compatibility.
+// ExtractorParam is the static configuration for the Extractor component.
+// Fully modularized into base settings and 5 sub-extraction tasks.
type ExtractorParam struct {
- // Modular sub-configs
- Keywords KeywordExtractConfig `json:"keywords,omitempty"`
- Questions QuestionExtractConfig `json:"questions,omitempty"`
- Tags TagExtractConfig `json:"tags,omitempty"`
- Summary SummaryExtractConfig `json:"summary,omitempty"`
- MetadataConfig MetadataExtractConfig `json:"metadata_config,omitempty"`
-
- // FieldName is the chunk key the LLM extraction result is written
- // to (Python: `self._param.field_name`). Optional — when empty,
- // auto_keywords or auto_questions may still be used.
- FieldName string `json:"field_name,omitempty"`
-
- // LLMID identifies the LLM model used for extraction. This is the
- // agent-side LLMParam.llm_id; on the ingestion side it is
- // resolved against the tenant's LLM provider registry.
+ // Base settings
LLMID string `json:"llm_id,omitempty"`
- // SystemPrompt is the optional system prompt override.
- SystemPrompt string `json:"system_prompt,omitempty"`
-
- // Prompt is the user-side template passed to the LLM.
- Prompt string `json:"prompt,omitempty"`
-
- // AutoKeywords enables automatic keyword extraction with a fixed
- // prompt. The value determines the top-N count. Legacy flat field.
- AutoKeywords int `json:"auto_keywords,omitempty"`
-
- // AutoQuestions enables automatic question generation with a fixed
- // prompt. The value determines the top-N count. Legacy flat field.
- AutoQuestions int `json:"auto_questions,omitempty"`
-
- // AutoTags enables tag assignment on chunks. Legacy flat field.
- AutoTags int `json:"auto_tags,omitempty"`
-
- // TagFileID references a tag-definition file stored in object
- // storage. Legacy flat field.
- TagFileID string `json:"tag_file_id,omitempty"`
-
- // EnableSummary enables summary extraction. Legacy flat field.
- EnableSummary int `json:"enable_summary,omitempty"`
-
- // EnableMetadata enables automatic structured metadata extraction. Legacy flat field.
- EnableMetadata int `json:"enable_metadata,omitempty"`
-
- // Metadata lists the target field definitions for
- // EnableMetadata (mirrors parser_config.metadata / built_in_metadata:
- // {key, type, description, enum}). When empty, EnableMetadata is a
- // no-op (nothing to extract).
- Metadata []common.MetadataFieldDef `json:"metadata,omitempty"`
+ // Modular sub-configs
+ Keywords KeywordExtractConfig `json:"keywords,omitempty"`
+ Questions QuestionExtractConfig `json:"questions,omitempty"`
+ Tags TagExtractConfig `json:"tags,omitempty"`
+ Summary SummaryExtractConfig `json:"summary,omitempty"`
+ Metadata MetadataExtractConfig `json:"metadata,omitempty"`
}
// Defaults returns the default ExtractorParam.
func (ExtractorParam) Defaults() ExtractorParam {
return ExtractorParam{
- FieldName: "",
- LLMID: "",
- SystemPrompt: "",
- Prompt: "",
- AutoKeywords: 0,
- AutoQuestions: 0,
- AutoTags: 0,
- TagFileID: "",
- EnableSummary: 0,
- EnableMetadata: 0,
- Metadata: nil,
+ LLMID: "",
}
}
@@ -192,11 +141,9 @@ type ExtractorOutputs struct {
// OutputFormat is always "chunks".
OutputFormat string `json:"output_format,omitempty"`
- // Chunks is the enriched chunk list. When the Extractor ran over
- // a non-empty input list, each chunk gains a new key named after
- // FieldName (e.g., field_name="summary" -> chunk["summary"]). When
- // the Extractor ran over an empty input, Chunks contains a single
- // entry with one key (FieldName) holding the LLM result.
+ // Chunks is the enriched chunk list. Each chunk is enriched with
+ // modular extraction fields (important_kwd, question_kwd, tag_kwd,
+ // summary, metadata).
Chunks []map[string]any `json:"chunks,omitempty"`
// Error is set when the component short-circuits with an error
diff --git a/internal/ingestion/component/schema/schema_test.go b/internal/ingestion/component/schema/schema_test.go
index f3f7509e65..c27ae5976d 100644
--- a/internal/ingestion/component/schema/schema_test.go
+++ b/internal/ingestion/component/schema/schema_test.go
@@ -504,8 +504,8 @@ func TestTokenizerOutputsJSONRoundTrip(t *testing.T) {
func TestExtractorParamDefaults(t *testing.T) {
p := ExtractorParam{}.Defaults()
- if p.FieldName != "" {
- t.Errorf("default field_name should be empty, got %q", p.FieldName)
+ if p.LLMID != "" {
+ t.Errorf("default llm_id should be empty, got %q", p.LLMID)
}
if err := p.Validate(); err != nil {
t.Fatalf("default ExtractorParam should pass Validate, got %v", err)
@@ -516,8 +516,8 @@ func TestExtractorParamValidate(t *testing.T) {
if err := (&ExtractorParam{}).Validate(); err != nil {
t.Fatalf("expected no error, got %v", err)
}
- if err := (&ExtractorParam{FieldName: "summary"}).Validate(); err != nil {
- t.Fatalf("Validate with field_name should pass, got %v", err)
+ if err := (&ExtractorParam{LLMID: "model@provider"}).Validate(); err != nil {
+ t.Fatalf("Validate should pass, got %v", err)
}
}
diff --git a/internal/ingestion/component/slice3_test.go b/internal/ingestion/component/slice3_test.go
deleted file mode 100644
index 4b64be01c4..0000000000
--- a/internal/ingestion/component/slice3_test.go
+++ /dev/null
@@ -1,75 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-// Slice 3 tests for port-rag-flow-pipeline-to-go.md Phase 5.
-//
-// Pins the Extractor prompt placeholder substitution.
-
-package component
-
-import (
- "strings"
- "testing"
-)
-
-// TestRenderExtractorPrompts_ReplacesAtChunks pins the
-// `{ComponentName:ParamName@chunks}` placeholder substitution.
-func TestRenderExtractorPrompts_ReplacesAtChunks(t *testing.T) {
- prompt := "Extract metadata from: {TitleChunker:FlatMiceFix@chunks}"
- ck := map[string]any{"text": "First chunk."}
- _, got := renderExtractorPrompts("", prompt, ck, "First chunk.")
- if strings.Contains(got, "{TitleChunker:FlatMiceFix@chunks}") {
- t.Errorf("placeholder not substituted: %q", got)
- }
- if !strings.Contains(got, "First chunk.") {
- t.Errorf("substitute missing chunk content: %q", got)
- }
- if n := strings.Count(got, "First chunk."); n != 1 {
- t.Errorf("chunk text appears %d times, want 1: %q", n, got)
- }
-}
-
-// TestRenderExtractorPrompts_LeavesUnknownPattern pins that unrecognized
-// placeholders are preserved verbatim.
-func TestRenderExtractorPrompts_LeavesUnknownPattern(t *testing.T) {
- prompt := "Extract metadata from: {unknown_placeholder}"
- _, got := renderExtractorPrompts("", prompt, nil, "")
- if got != prompt {
- t.Errorf("unknown placeholder: pattern should be preserved\n got: %q\n want: %q", got, prompt)
- }
-}
-
-// TestRenderExtractorPrompts_NoPlaceholderAppendsChunkText pins the
-// fallback append behavior when the prompt carries no body placeholder.
-func TestRenderExtractorPrompts_NoPlaceholderAppendsChunkText(t *testing.T) {
- prompt := "Plain prompt with no substitution."
- ck := map[string]any{"text": "my chunk text"}
- _, got := renderExtractorPrompts("", prompt, ck, "my chunk text")
- want := "Plain prompt with no substitution.\n\nmy chunk text"
- if got != want {
- t.Errorf("got %q, want %q", got, want)
- }
-}
-
-// TestRenderExtractorPrompts_SkipsEmptyChunkText pins that an empty
-// chunkText does not append a trailing blank line.
-func TestRenderExtractorPrompts_SkipsEmptyChunkText(t *testing.T) {
- prompt := "Plain prompt"
- _, got := renderExtractorPrompts("", prompt, map[string]any{}, "")
- if got != prompt {
- t.Errorf("empty chunkText should not alter prompt\n got: %q\n want: %q", got, prompt)
- }
-}
diff --git a/internal/ingestion/pipeline/component_params.go b/internal/ingestion/pipeline/component_params.go
index fcb975ff91..f9c73e3e2b 100644
--- a/internal/ingestion/pipeline/component_params.go
+++ b/internal/ingestion/pipeline/component_params.go
@@ -38,6 +38,16 @@ func ExtractAllComponentParams(dslJSON []byte) ([]ComponentParamsSchema, error)
return nil, fmt.Errorf("ExtractAllComponentParams: DSL has no components")
}
+ extractorCount := 0
+ for cpnID, comp := range dsl.Components {
+ if isExtractorComponent(cpnID, comp.Obj.ComponentName) {
+ extractorCount++
+ }
+ }
+ if extractorCount > 1 {
+ return nil, fmt.Errorf("ExtractAllComponentParams: at most 1 Extractor component is allowed, found %d", extractorCount)
+ }
+
out := make([]ComponentParamsSchema, 0, len(dsl.Components))
for cpnID, comp := range dsl.Components {
params := make(map[string]any, len(comp.Obj.Params))
diff --git a/internal/ingestion/pipeline/component_params_test.go b/internal/ingestion/pipeline/component_params_test.go
index ce8798f0a8..42244a2e14 100644
--- a/internal/ingestion/pipeline/component_params_test.go
+++ b/internal/ingestion/pipeline/component_params_test.go
@@ -237,3 +237,34 @@ func TestExtractAllComponentParams_AllBuiltinTemplates(t *testing.T) {
}
t.Logf("tested %d builtin templates", tested)
}
+
+func TestExtractAllComponentParams_DisallowMultipleExtractors(t *testing.T) {
+ dsl := map[string]any{
+ "components": map[string]any{
+ "Extractor:One": map[string]any{
+ "obj": map[string]any{
+ "component_name": "Extractor",
+ "params": map[string]any{},
+ },
+ },
+ "Extractor:Two": map[string]any{
+ "obj": map[string]any{
+ "component_name": "Extractor",
+ "params": map[string]any{},
+ },
+ },
+ },
+ }
+ dslJSON, err := json.Marshal(dsl)
+ if err != nil {
+ t.Fatalf("marshal dsl: %v", err)
+ }
+
+ _, err = ExtractAllComponentParams(dslJSON)
+ if err == nil {
+ t.Fatal("expected error for multiple Extractor components, got nil")
+ }
+ if !strings.Contains(err.Error(), "at most 1 Extractor") {
+ t.Errorf("expected at most 1 Extractor error message, got: %v", err)
+ }
+}
diff --git a/internal/ingestion/pipeline/pipeline.go b/internal/ingestion/pipeline/pipeline.go
index f43cf7ae50..8f50a92f69 100644
--- a/internal/ingestion/pipeline/pipeline.go
+++ b/internal/ingestion/pipeline/pipeline.go
@@ -152,6 +152,9 @@ func NewPipelineFromDSL(dsl []byte, taskID string, opts ...PipelineOption) (*Pip
if err != nil {
return nil, fmt.Errorf("pipeline: decode canvas DSL: %w", err)
}
+ if err := ValidatePipeline(cnv); err != nil {
+ return nil, err
+ }
// Capture the canonical canvas DSL bytes for the resume-time DSL
// fingerprint. json.Marshal sorts map keys, so this is stable across
// re-decodes of the same logical DSL (formatting-independent).
@@ -170,6 +173,24 @@ func NewPipelineFromDSL(dsl []byte, taskID string, opts ...PipelineOption) (*Pip
return p, nil
}
+// ValidatePipeline enforces ingestion pipeline constraints.
+// Specifically, at most one Extractor component is permitted in the graph.
+func ValidatePipeline(cnv *canvas.Canvas) error {
+ if cnv == nil {
+ return nil
+ }
+ extractorCount := 0
+ for id, comp := range cnv.Components {
+ if isExtractorComponent(id, comp.Obj.ComponentName) {
+ extractorCount++
+ }
+ }
+ if extractorCount > 1 {
+ return fmt.Errorf("pipeline validation error: at most 1 Extractor component is allowed, found %d", extractorCount)
+ }
+ return nil
+}
+
// WithComponentFactory installs an instance-scoped factory override for this
// pipeline. It is used during canvas compilation so one pipeline run can
// construct task-specific component instances without mutating the process-wide
diff --git a/internal/ingestion/pipeline/pipeline_params.go b/internal/ingestion/pipeline/pipeline_params.go
index d6784fe80c..82312e8cce 100644
--- a/internal/ingestion/pipeline/pipeline_params.go
+++ b/internal/ingestion/pipeline/pipeline_params.go
@@ -2,19 +2,90 @@ package pipeline
import (
"fmt"
+ "reflect"
"strings"
"ragflow/internal/common"
"ragflow/internal/entity"
+ "ragflow/internal/ingestion/component/schema"
"go.uber.org/zap"
)
+// llmRuntimeParamKeys contains generic LLM hyper-parameters and UI switches common to LLM-based components.
+var llmRuntimeParamKeys = map[string]struct{}{
+ "llm_id": {},
+ "temperature": {},
+ "top_p": {},
+ "max_tokens": {},
+ "presence_penalty": {},
+ "frequency_penalty": {},
+ "outputs": {},
+ "temperatureEnabled": {},
+ "topPEnabled": {},
+ "presencePenaltyEnabled": {},
+ "frequencyPenaltyEnabled": {},
+ "maxTokensEnabled": {},
+}
+
+// extractorValidParamKeys is dynamically derived from schema.ExtractorParam + LLM hyper-parameters.
+var extractorValidParamKeys = func() map[string]struct{} {
+ keys := extractJSONTags(schema.ExtractorParam{})
+ for k := range llmRuntimeParamKeys {
+ keys[k] = struct{}{}
+ }
+ return keys
+}()
+
+// extractJSONTags returns all top-level json tag names from a struct.
+func extractJSONTags(v any) map[string]struct{} {
+ tags := make(map[string]struct{})
+ t := reflect.TypeOf(v)
+ if t == nil {
+ return tags
+ }
+ if t.Kind() == reflect.Ptr {
+ t = t.Elem()
+ }
+ if t.Kind() != reflect.Struct {
+ return tags
+ }
+ for i := 0; i < t.NumField(); i++ {
+ tag := t.Field(i).Tag.Get("json")
+ if tag == "" || tag == "-" {
+ continue
+ }
+ name := strings.Split(tag, ",")[0]
+ if name != "" {
+ tags[name] = struct{}{}
+ }
+ }
+ return tags
+}
+
+// isExtractorComponent returns true if the component name or component ID
+// indicates an Extractor component.
+func isExtractorComponent(cpnID, componentName string) bool {
+ lowerName := strings.ToLower(componentName)
+ lowerID := strings.ToLower(cpnID)
+ return lowerName == "extractor" ||
+ strings.HasPrefix(lowerID, "extractor:") ||
+ strings.HasPrefix(lowerID, "extractor_")
+}
+
+// getComponentParamWhitelist returns the dynamic parameter whitelist for a component type.
+func getComponentParamWhitelist(cpnID string) (map[string]struct{}, bool) {
+ if isExtractorComponent(cpnID, "") {
+ return extractorValidParamKeys, true
+ }
+ return nil, false
+}
+
// CleanComponentParams filters rawConfig against the DSL schema given by dslJSON.
// Keys containing ':' are treated as component IDs; they are kept only when both
-// the cpnID AND the param name exist in the DSL schema. Keys without ':' (legacy
-// flat fields such as chunk_token_num, image_context_size) are dropped with a
-// warning — they do not belong in the new component-params world.
+// the cpnID AND the param name exist in the DSL schema or the component's dynamic
+// parameter schema (e.g. Extractor modular features). Keys without ':' (legacy
+// flat fields) are dropped with a warning.
func CleanComponentParams(dslJSON []byte, rawConfig map[string]interface{}) map[string]interface{} {
schemas, err := ExtractAllComponentParams(dslJSON)
if err != nil {
@@ -49,18 +120,22 @@ func CleanComponentParams(dslJSON []byte, rawConfig map[string]interface{}) map[
if !ok {
continue
}
+ dynamicWhitelist, hasDynamic := getComponentParamWhitelist(key)
+ if hasDynamic {
+ params = NormalizeExtractorParams(params)
+ }
cleaned := make(map[string]any, len(params))
for pk, pv := range params {
if _, ok := validKeys[pk]; ok {
cleaned[pk] = pv
continue
}
- // Extractor params are valid even when a builtin template does
- // not declare them explicitly in params: {} (the frontend stores
- // them on the node params); keep them through the filter.
- if isExtractorComponent(key) && isExtractorParam(pk) {
- cleaned[pk] = pv
- continue
+ // Extractor and dynamic components: check reflection-driven whitelist
+ if hasDynamic {
+ if _, ok := dynamicWhitelist[pk]; ok {
+ cleaned[pk] = pv
+ continue
+ }
}
common.Warn("CleanComponentParams: dropping unknown param",
zap.String("cpnID", key), zap.String("param", pk))
@@ -72,6 +147,64 @@ func CleanComponentParams(dslJSON []byte, rawConfig map[string]interface{}) map[
return result
}
+// NormalizeExtractorParams normalizes a raw Extractor parameters map into the canonical modular format.
+func NormalizeExtractorParams(raw map[string]any) map[string]any {
+ if raw == nil {
+ return map[string]any{}
+ }
+ out := make(map[string]any, len(raw))
+ for k, v := range raw {
+ out[k] = v
+ }
+
+ // 1. Keywords
+ if kwRaw, ok := out["keywords"].(map[string]any); ok {
+ kw := make(map[string]any, len(kwRaw))
+ for k, v := range kwRaw {
+ kw[k] = v
+ }
+ out["keywords"] = kw
+ }
+
+ // 2. Questions
+ if qRaw, ok := out["questions"].(map[string]any); ok {
+ q := make(map[string]any, len(qRaw))
+ for k, v := range qRaw {
+ q[k] = v
+ }
+ out["questions"] = q
+ }
+
+ // 3. Tags
+ if tagRaw, ok := out["tags"].(map[string]any); ok {
+ tag := make(map[string]any, len(tagRaw))
+ for k, v := range tagRaw {
+ tag[k] = v
+ }
+ out["tags"] = tag
+ }
+
+ // 4. Summary
+ if sumRaw, ok := out["summary"].(map[string]any); ok {
+ sum := make(map[string]any, len(sumRaw))
+ for k, v := range sumRaw {
+ sum[k] = v
+ }
+ out["summary"] = sum
+ }
+
+ // 5. Metadata
+ if metaRaw, ok := out["metadata"].(map[string]any); ok {
+ meta := make(map[string]any, len(metaRaw))
+ for k, v := range metaRaw {
+ meta[k] = v
+ }
+ out["metadata"] = meta
+ }
+
+ return out
+}
+
// BuildParserConfig builds the final parser_config by starting from the DSL
// defaults for every component, then overlaying the cleaned incoming overrides.
// This ensures all components from the current pipeline are present while
@@ -156,28 +289,3 @@ func ResolveComponentParamsDefaultsFromIDs(parserID string, pipelineID *string)
}
return out, nil
}
-
-// isExtractorComponent reports whether a component id is an Extractor node
-// (component ids are "<ComponentName>:<name>" or "<ComponentName>_<name>").
-func isExtractorComponent(cpnID string) bool {
- lower := strings.ToLower(cpnID)
- return strings.HasPrefix(lower, "extractor:") || strings.HasPrefix(lower, "extractor_")
-}
-
-// isExtractorParam reports whether a param belongs to the Extractor component
-// (modular sub-configs, legacy flat fields, LLM settings, or UI state flags).
-func isExtractorParam(param string) bool {
- switch param {
- case "keywords", "questions", "tags", "summary", "metadata_config",
- "auto_keywords", "keywords_sys_prompt",
- "auto_questions", "questions_sys_prompt",
- "auto_tags", "tag_file_id",
- "enable_summary", "sys_prompt", "system_prompt", "prompt", "prompts", "field_name",
- "enable_metadata", "metadata", "built_in_metadata",
- "llm_id", "temperature", "top_p", "max_tokens", "presence_penalty", "frequency_penalty",
- "temperatureEnabled", "topPEnabled", "maxTokensEnabled", "presencePenaltyEnabled", "frequencyPenaltyEnabled",
- "outputs":
- return true
- }
- return false
-}
diff --git a/internal/ingestion/pipeline/pipeline_params_test.go b/internal/ingestion/pipeline/pipeline_params_test.go
index 1f544dc05d..fbe0aee400 100644
--- a/internal/ingestion/pipeline/pipeline_params_test.go
+++ b/internal/ingestion/pipeline/pipeline_params_test.go
@@ -239,15 +239,21 @@ func TestBuildParserConfig_BuiltinExtractorKeepsTagFileID(t *testing.T) {
}
checked++
overrides := map[string]any{
- s.CpnID: map[string]any{"tag_file_id": "file-123"},
+ s.CpnID: map[string]any{
+ "tags": map[string]any{"top_n": 5, "tag_file_id": "file-123"},
+ },
}
result := BuildParserConfig(dslJSON, overrides)
params, ok := result[s.CpnID].(map[string]any)
if !ok {
t.Fatalf("template %q: expected component %q in result", ref, s.CpnID)
}
- if params["tag_file_id"] != "file-123" {
- t.Errorf("template %q: expected tag_file_id to survive BuildParserConfig, got %v", ref, params["tag_file_id"])
+ var tagFileID any
+ if tags, ok := params["tags"].(map[string]any); ok {
+ tagFileID = tags["tag_file_id"]
+ }
+ if tagFileID != "file-123" {
+ t.Errorf("template %q: expected tag_file_id to survive BuildParserConfig, got %v", ref, tagFileID)
}
}
}
@@ -335,8 +341,11 @@ func TestBuildParserConfig_BuiltinExtractorKeepsBuiltInMetadata(t *testing.T) {
checked++
overrides := map[string]any{
s.CpnID: map[string]any{
- "built_in_metadata": []any{
- map[string]any{"key": "update_time", "type": "time"},
+ "metadata": map[string]any{
+ "enabled": true,
+ "built_in_metadata": []any{
+ map[string]any{"key": "update_time", "type": "time"},
+ },
},
},
}
@@ -345,8 +354,8 @@ func TestBuildParserConfig_BuiltinExtractorKeepsBuiltInMetadata(t *testing.T) {
if !ok {
t.Fatalf("template %q: expected component %q in result", ref, s.CpnID)
}
- if _, ok := params["built_in_metadata"]; !ok {
- t.Errorf("template %q: built_in_metadata dropped by CleanComponentParams; add \"built_in_metadata\": [] to the extractor node params", ref)
+ if _, ok := params["metadata"]; !ok {
+ t.Errorf("template %q: metadata dropped by CleanComponentParams", ref)
}
}
}
@@ -378,7 +387,7 @@ func TestCleanComponentParams_KeepsModularExtractorParams(t *testing.T) {
"enabled": true,
"system_prompt": "summary prompt",
},
- "metadata_config": map[string]any{
+ "metadata": map[string]any{
"enabled": true,
"metadata": []any{
map[string]any{"key": "category", "type": "string"},
@@ -387,9 +396,9 @@ func TestCleanComponentParams_KeepsModularExtractorParams(t *testing.T) {
"keywords": map[string]any{
"top_n": 5,
},
- "enable_summary": 1,
"llm_id": "gpt-4",
"temperature": 0.7,
+ "enable_summary": 1, // legacy flat param should be dropped
},
}
@@ -402,8 +411,8 @@ func TestCleanComponentParams_KeepsModularExtractorParams(t *testing.T) {
if sum, ok := ext["summary"].(map[string]any); !ok || sum["enabled"] != true {
t.Errorf("expected summary.enabled == true, got: %v", ext["summary"])
}
- if meta, ok := ext["metadata_config"].(map[string]any); !ok || meta["enabled"] != true {
- t.Errorf("expected metadata_config.enabled == true, got: %v", ext["metadata_config"])
+ if meta, ok := ext["metadata"].(map[string]any); !ok || meta["enabled"] != true {
+ t.Errorf("expected metadata.enabled == true, got: %v", ext["metadata"])
}
if kw, ok := ext["keywords"].(map[string]any); !ok || kw["top_n"] != 5 {
t.Errorf("expected keywords.top_n == 5, got: %v", ext["keywords"])
@@ -411,4 +420,172 @@ func TestCleanComponentParams_KeepsModularExtractorParams(t *testing.T) {
if ext["llm_id"] != "gpt-4" {
t.Errorf("expected llm_id == gpt-4, got: %v", ext["llm_id"])
}
+ if _, ok := ext["enable_summary"]; ok {
+ t.Errorf("expected legacy flat field enable_summary to be dropped, got: %v", ext["enable_summary"])
+ }
+}
+
+func TestCleanComponentParams_ExtractorBuiltInMetadataRetained(t *testing.T) {
+ dslJSON := []byte(`{
+ "components": {
+ "Extractor:AutoExtractDefault": {
+ "obj": {
+ "component_name": "Extractor",
+ "params": {
+ "outputs": {},
+ "metadata": {}
+ }
+ }
+ }
+ }
+ }`)
+
+ rawConfig := map[string]any{
+ "Extractor:AutoExtractDefault": map[string]any{
+ "metadata": map[string]any{
+ "enabled": true,
+ "metadata": []map[string]any{
+ {"key": "custom_field", "type": "string"},
+ },
+ "built_in_metadata": []map[string]any{
+ {"key": "doc_name", "type": "string"},
+ },
+ },
+ },
+ }
+
+ result := CleanComponentParams(dslJSON, rawConfig)
+ ext, ok := result["Extractor:AutoExtractDefault"].(map[string]any)
+ if !ok {
+ t.Fatalf("expected Extractor:AutoExtractDefault in result, got: %v", result)
+ }
+
+ meta, ok := ext["metadata"].(map[string]any)
+ if !ok {
+ t.Fatalf("expected metadata object, got: %#v", ext["metadata"])
+ }
+ if meta["enabled"] != true {
+ t.Errorf("expected enabled == true, got: %#v", meta["enabled"])
+ }
+ mFields, ok := meta["metadata"].([]map[string]any)
+ if !ok || len(mFields) != 1 {
+ t.Errorf("expected 1 metadata field, got: %#v", meta["metadata"])
+ }
+ bFields, ok := meta["built_in_metadata"].([]map[string]any)
+ if !ok || len(bFields) != 1 {
+ t.Errorf("expected 1 built_in_metadata field, got: %#v", meta["built_in_metadata"])
+ }
+}
+
+func TestNormalizeExtractorParams_TableDriven(t *testing.T) {
+ tests := []struct {
+ name string
+ input map[string]any
+ validate func(t *testing.T, out map[string]any)
+ }{
+ {
+ name: "nil input returns empty map",
+ input: nil,
+ validate: func(t *testing.T, out map[string]any) {
+ if len(out) != 0 {
+ t.Errorf("expected empty map for nil input, got %#v", out)
+ }
+ },
+ },
+ {
+ name: "modular keywords preserved",
+ input: map[string]any{
+ "keywords": map[string]any{
+ "top_n": 5,
+ "system_prompt": "custom kw prompt",
+ },
+ },
+ validate: func(t *testing.T, out map[string]any) {
+ kw, ok := out["keywords"].(map[string]any)
+ if !ok || kw["top_n"] != 5 || kw["system_prompt"] != "custom kw prompt" {
+ t.Errorf("keywords mismatch: %#v", out["keywords"])
+ }
+ },
+ },
+ {
+ name: "modular questions preserved",
+ input: map[string]any{
+ "questions": map[string]any{
+ "top_n": 3,
+ "system_prompt": "custom q prompt",
+ },
+ },
+ validate: func(t *testing.T, out map[string]any) {
+ q, ok := out["questions"].(map[string]any)
+ if !ok || q["top_n"] != 3 || q["system_prompt"] != "custom q prompt" {
+ t.Errorf("questions mismatch: %#v", out["questions"])
+ }
+ },
+ },
+ {
+ name: "modular tags preserved",
+ input: map[string]any{
+ "tags": map[string]any{
+ "top_n": 7,
+ "tag_file_id": "file-123",
+ },
+ },
+ validate: func(t *testing.T, out map[string]any) {
+ tag, ok := out["tags"].(map[string]any)
+ if !ok || tag["top_n"] != 7 || tag["tag_file_id"] != "file-123" {
+ t.Errorf("tags mismatch: %#v", out["tags"])
+ }
+ },
+ },
+ {
+ name: "modular summary preserved",
+ input: map[string]any{
+ "summary": map[string]any{
+ "enabled": true,
+ "system_prompt": "summary prompt",
+ },
+ },
+ validate: func(t *testing.T, out map[string]any) {
+ sum, ok := out["summary"].(map[string]any)
+ if !ok || sum["enabled"] != true || sum["system_prompt"] != "summary prompt" {
+ t.Errorf("summary mismatch: %#v", out["summary"])
+ }
+ },
+ },
+ {
+ name: "modular metadata preserved",
+ input: map[string]any{
+ "metadata": map[string]any{
+ "enabled": true,
+ "metadata": []map[string]any{
+ {"key": "cat", "type": "string"},
+ },
+ "built_in_metadata": []map[string]any{
+ {"key": "file_name", "type": "string"},
+ },
+ },
+ },
+ validate: func(t *testing.T, out map[string]any) {
+ meta, ok := out["metadata"].(map[string]any)
+ if !ok || meta["enabled"] != true {
+ t.Fatalf("expected enabled == true, got %#v", out["metadata"])
+ }
+ mList, ok := meta["metadata"].([]map[string]any)
+ if !ok || len(mList) != 1 {
+ t.Errorf("metadata slice mismatch: %#v", meta["metadata"])
+ }
+ bList, ok := meta["built_in_metadata"].([]map[string]any)
+ if !ok || len(bList) != 1 {
+ t.Errorf("built_in_metadata slice mismatch: %#v", meta["built_in_metadata"])
+ }
+ },
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ out := NormalizeExtractorParams(tc.input)
+ tc.validate(t, out)
+ })
+ }
}
diff --git a/internal/ingestion/pipeline/pipeline_test.go b/internal/ingestion/pipeline/pipeline_test.go
index 1aa9970b22..79b1ad4731 100644
--- a/internal/ingestion/pipeline/pipeline_test.go
+++ b/internal/ingestion/pipeline/pipeline_test.go
@@ -20,6 +20,7 @@ import (
"context"
"errors"
"fmt"
+ "strings"
"sync"
"testing"
"time"
@@ -909,3 +910,25 @@ func TestRunPlain_WithTracker_Error(t *testing.T) {
t.Fatal("expected stage error, got nil")
}
}
+
+func TestValidatePipeline_DisallowMultipleExtractors(t *testing.T) {
+ dsl := []byte(`{
+ "dsl": {
+ "components": {
+ "begin": {"obj": {"component_name": "Begin", "params": {}}, "downstream": ["Extractor:One"]},
+ "Extractor:One": {"obj": {"component_name": "Extractor", "params": {}}, "upstream": ["begin"], "downstream": ["Extractor:Two"]},
+ "Extractor:Two": {"obj": {"component_name": "Extractor", "params": {}}, "upstream": ["Extractor:One"]}
+ },
+ "path": ["begin", "Extractor:One", "Extractor:Two"],
+ "graph": {"nodes": []}
+ }
+ }`)
+
+ _, err := NewPipelineFromDSL(dsl, "test-task-dup-ext")
+ if err == nil {
+ t.Fatal("expected validation error for multiple Extractor components, got nil")
+ }
+ if !strings.Contains(err.Error(), "at most 1 Extractor component is allowed") {
+ t.Errorf("expected at most 1 Extractor error message, got: %v", err)
+ }
+}
diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_audio.json b/internal/ingestion/pipeline/template/ingestion_pipeline_audio.json
index 90b6e82366..924cb74476 100644
--- a/internal/ingestion/pipeline/template/ingestion_pipeline_audio.json
+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_audio.json
@@ -125,20 +125,25 @@
"obj": {
"component_name": "Extractor",
"params": {
- "field_name": "",
- "outputs": {
- "chunks": {
- "type": "Array<Object>",
- "value": []
- }
- },
- "auto_keywords": 0,
- "auto_questions": 0,
"llm_id": "",
- "auto_tags": 0,
- "tag_file_id": "",
- "enable_metadata": 0,
- "metadata": []
+ "keywords": {
+ "top_n": 0
+ },
+ "questions": {
+ "top_n": 0
+ },
+ "tags": {
+ "top_n": 0,
+ "tag_file_id": ""
+ },
+ "summary": {
+ "enabled": false
+ },
+ "metadata": {
+ "enabled": false,
+ "metadata": [],
+ "built_in_metadata": []
+ }
}
}
}
diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_book.json b/internal/ingestion/pipeline/template/ingestion_pipeline_book.json
index ddc987eabe..09fa90bdbb 100644
--- a/internal/ingestion/pipeline/template/ingestion_pipeline_book.json
+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_book.json
@@ -186,20 +186,25 @@
"obj": {
"component_name": "Extractor",
"params": {
- "field_name": "",
- "outputs": {
- "chunks": {
- "type": "Array<Object>",
- "value": []
- }
- },
- "auto_keywords": 0,
- "auto_questions": 0,
"llm_id": "",
- "auto_tags": 0,
- "tag_file_id": "",
- "enable_metadata": 0,
- "metadata": []
+ "keywords": {
+ "top_n": 0
+ },
+ "questions": {
+ "top_n": 0
+ },
+ "tags": {
+ "top_n": 0,
+ "tag_file_id": ""
+ },
+ "summary": {
+ "enabled": false
+ },
+ "metadata": {
+ "enabled": false,
+ "metadata": [],
+ "built_in_metadata": []
+ }
}
}
}
diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_email.json b/internal/ingestion/pipeline/template/ingestion_pipeline_email.json
index 09c58c291e..a901ef0795 100644
--- a/internal/ingestion/pipeline/template/ingestion_pipeline_email.json
+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_email.json
@@ -130,20 +130,25 @@
"obj": {
"component_name": "Extractor",
"params": {
- "field_name": "",
- "outputs": {
- "chunks": {
- "type": "Array<Object>",
- "value": []
- }
- },
- "auto_keywords": 0,
- "auto_questions": 0,
"llm_id": "",
- "auto_tags": 0,
- "tag_file_id": "",
- "enable_metadata": 0,
- "metadata": []
+ "keywords": {
+ "top_n": 0
+ },
+ "questions": {
+ "top_n": 0
+ },
+ "tags": {
+ "top_n": 0,
+ "tag_file_id": ""
+ },
+ "summary": {
+ "enabled": false
+ },
+ "metadata": {
+ "enabled": false,
+ "metadata": [],
+ "built_in_metadata": []
+ }
}
}
}
diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_general.json b/internal/ingestion/pipeline/template/ingestion_pipeline_general.json
index 22d0d6fe7e..3374a6696f 100644
--- a/internal/ingestion/pipeline/template/ingestion_pipeline_general.json
+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_general.json
@@ -214,20 +214,25 @@
"obj": {
"component_name": "Extractor",
"params": {
- "field_name": "",
- "outputs": {
- "chunks": {
- "type": "Array<Object>",
- "value": []
- }
- },
- "auto_keywords": 0,
- "auto_questions": 0,
"llm_id": "",
- "auto_tags": 0,
- "tag_file_id": "",
- "enable_metadata": 0,
- "metadata": []
+ "keywords": {
+ "top_n": 0
+ },
+ "questions": {
+ "top_n": 0
+ },
+ "tags": {
+ "top_n": 0,
+ "tag_file_id": ""
+ },
+ "summary": {
+ "enabled": false
+ },
+ "metadata": {
+ "enabled": false,
+ "metadata": [],
+ "built_in_metadata": []
+ }
}
}
}
diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_laws.json b/internal/ingestion/pipeline/template/ingestion_pipeline_laws.json
index b08be5d874..b1b80c2acd 100644
--- a/internal/ingestion/pipeline/template/ingestion_pipeline_laws.json
+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_laws.json
@@ -199,20 +199,25 @@
"obj": {
"component_name": "Extractor",
"params": {
- "field_name": "",
- "outputs": {
- "chunks": {
- "type": "Array<Object>",
- "value": []
- }
- },
- "auto_keywords": 0,
- "auto_questions": 0,
"llm_id": "",
- "auto_tags": 0,
- "tag_file_id": "",
- "enable_metadata": 0,
- "metadata": []
+ "keywords": {
+ "top_n": 0
+ },
+ "questions": {
+ "top_n": 0
+ },
+ "tags": {
+ "top_n": 0,
+ "tag_file_id": ""
+ },
+ "summary": {
+ "enabled": false
+ },
+ "metadata": {
+ "enabled": false,
+ "metadata": [],
+ "built_in_metadata": []
+ }
}
}
}
diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json b/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json
index 1faad567e9..2809b48305 100644
--- a/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json
+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json
@@ -166,20 +166,25 @@
"obj": {
"component_name": "Extractor",
"params": {
- "field_name": "",
- "outputs": {
- "chunks": {
- "type": "Array<Object>",
- "value": []
- }
- },
- "auto_keywords": 0,
- "auto_questions": 0,
"llm_id": "",
- "auto_tags": 0,
- "tag_file_id": "",
- "enable_metadata": 0,
- "metadata": []
+ "keywords": {
+ "top_n": 0
+ },
+ "questions": {
+ "top_n": 0
+ },
+ "tags": {
+ "top_n": 0,
+ "tag_file_id": ""
+ },
+ "summary": {
+ "enabled": false
+ },
+ "metadata": {
+ "enabled": false,
+ "metadata": [],
+ "built_in_metadata": []
+ }
}
}
}
diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_one.json b/internal/ingestion/pipeline/template/ingestion_pipeline_one.json
index 3406161af5..0bcfb58a44 100644
--- a/internal/ingestion/pipeline/template/ingestion_pipeline_one.json
+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_one.json
@@ -179,20 +179,25 @@
"obj": {
"component_name": "Extractor",
"params": {
- "field_name": "",
- "outputs": {
- "chunks": {
- "type": "Array<Object>",
- "value": []
- }
- },
- "auto_keywords": 0,
- "auto_questions": 0,
"llm_id": "",
- "auto_tags": 0,
- "tag_file_id": "",
- "enable_metadata": 0,
- "metadata": []
+ "keywords": {
+ "top_n": 0
+ },
+ "questions": {
+ "top_n": 0
+ },
+ "tags": {
+ "top_n": 0,
+ "tag_file_id": ""
+ },
+ "summary": {
+ "enabled": false
+ },
+ "metadata": {
+ "enabled": false,
+ "metadata": [],
+ "built_in_metadata": []
+ }
}
}
}
diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_paper.json b/internal/ingestion/pipeline/template/ingestion_pipeline_paper.json
index 07a49d26e2..a679e801b6 100644
--- a/internal/ingestion/pipeline/template/ingestion_pipeline_paper.json
+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_paper.json
@@ -147,20 +147,25 @@
"obj": {
"component_name": "Extractor",
"params": {
- "field_name": "",
- "outputs": {
- "chunks": {
- "type": "Array<Object>",
- "value": []
- }
- },
- "auto_keywords": 0,
- "auto_questions": 0,
"llm_id": "",
- "auto_tags": 0,
- "tag_file_id": "",
- "enable_metadata": 0,
- "metadata": []
+ "keywords": {
+ "top_n": 0
+ },
+ "questions": {
+ "top_n": 0
+ },
+ "tags": {
+ "top_n": 0,
+ "tag_file_id": ""
+ },
+ "summary": {
+ "enabled": false
+ },
+ "metadata": {
+ "enabled": false,
+ "metadata": [],
+ "built_in_metadata": []
+ }
}
}
}
diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_picture.json b/internal/ingestion/pipeline/template/ingestion_pipeline_picture.json
index effb4e5b7d..5d6fe299bd 100644
--- a/internal/ingestion/pipeline/template/ingestion_pipeline_picture.json
+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_picture.json
@@ -139,20 +139,25 @@
"obj": {
"component_name": "Extractor",
"params": {
- "field_name": "",
- "outputs": {
- "chunks": {
- "type": "Array<Object>",
- "value": []
- }
- },
- "auto_keywords": 0,
- "auto_questions": 0,
"llm_id": "",
- "auto_tags": 0,
- "tag_file_id": "",
- "enable_metadata": 0,
- "metadata": []
+ "keywords": {
+ "top_n": 0
+ },
+ "questions": {
+ "top_n": 0
+ },
+ "tags": {
+ "top_n": 0,
+ "tag_file_id": ""
+ },
+ "summary": {
+ "enabled": false
+ },
+ "metadata": {
+ "enabled": false,
+ "metadata": [],
+ "built_in_metadata": []
+ }
}
}
}
diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_presentation.json b/internal/ingestion/pipeline/template/ingestion_pipeline_presentation.json
index b4746fff1c..c10eab6733 100644
--- a/internal/ingestion/pipeline/template/ingestion_pipeline_presentation.json
+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_presentation.json
@@ -125,20 +125,25 @@
"obj": {
"component_name": "Extractor",
"params": {
- "field_name": "",
- "outputs": {
- "chunks": {
- "type": "Array<Object>",
- "value": []
- }
- },
- "auto_keywords": 0,
- "auto_questions": 0,
"llm_id": "",
- "auto_tags": 0,
- "tag_file_id": "",
- "enable_metadata": 0,
- "metadata": []
+ "keywords": {
+ "top_n": 0
+ },
+ "questions": {
+ "top_n": 0
+ },
+ "tags": {
+ "top_n": 0,
+ "tag_file_id": ""
+ },
+ "summary": {
+ "enabled": false
+ },
+ "metadata": {
+ "enabled": false,
+ "metadata": [],
+ "built_in_metadata": []
+ }
}
}
}
diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.go
index d63fba4611..3d224c8121 100644
--- a/internal/ingestion/task/pipeline_executor.go
+++ b/internal/ingestion/task/pipeline_executor.go
@@ -309,7 +309,8 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map
func builtInMetadataFromParserConfig(parserConfig entity.JSONMap) ([]any, bool) {
var extractorKeys []string
for k := range parserConfig {
- if strings.HasPrefix(k, "Extractor") {
+ lower := strings.ToLower(k)
+ if strings.HasPrefix(lower, "extractor:") || strings.HasPrefix(lower, "extractor_") {
extractorKeys = append(extractorKeys, k)
}
}
@@ -318,6 +319,12 @@ func builtInMetadataFromParserConfig(parserConfig entity.JSONMap) ([]any, bool)
for _, k := range extractorKeys {
nodeRaw := parserConfig[k]
if node, ok := nodeRaw.(map[string]any); ok {
+ if metaObj, ok := node["metadata"].(map[string]any); ok {
+ enabled := parserConfigBool(metaObj["enabled"])
+ if arr, ok := metaObj["built_in_metadata"].([]any); ok && len(arr) > 0 {
+ return arr, enabled
+ }
+ }
if metaConf, ok := node["metadata_config"].(map[string]any); ok {
enabled := parserConfigBool(metaConf["enabled"])
if arr, ok := metaConf["built_in_metadata"].([]any); ok && len(arr) > 0 {
@@ -330,14 +337,18 @@ func builtInMetadataFromParserConfig(parserConfig entity.JSONMap) ([]any, bool)
}
}
}
+ if metaObj, ok := parserConfig["metadata"].(map[string]any); ok {
+ if arr, ok := metaObj["built_in_metadata"].([]any); ok && len(arr) > 0 {
+ return arr, true
+ }
+ }
if metaConf, ok := parserConfig["metadata_config"].(map[string]any); ok {
- enabled := parserConfigBool(metaConf["enabled"])
if arr, ok := metaConf["built_in_metadata"].([]any); ok && len(arr) > 0 {
- return arr, enabled
+ return arr, true
}
}
if arr, ok := parserConfig["built_in_metadata"].([]any); ok && len(arr) > 0 {
- return arr, parserConfigBool(parserConfig["enable_metadata"])
+ return arr, true
}
return nil, false
}
diff --git a/internal/ingestion/task/pipeline_executor_defaults_test.go b/internal/ingestion/task/pipeline_executor_defaults_test.go
index 6d9fcca7be..0330babc95 100644
--- a/internal/ingestion/task/pipeline_executor_defaults_test.go
+++ b/internal/ingestion/task/pipeline_executor_defaults_test.go
@@ -38,16 +38,16 @@ import (
// Comparison is done per-component (see assertComponentsMatch), so the JSON
// serialization order inside each entry is irrelevant.
var builtinComponentParamsGolden = map[string]string{
- "audio": "{\"File\": {}, \"Parser:SongsFillAir\": {\"audio\": {\"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"aac\", \"aiff\", \"ape\", \"au\", \"da\", \"flac\", \"midi\", \"mp3\", \"ogg\", \"oggvorbis\", \"realaudio\", \"vqf\", \"wav\", \"wave\", \"wma\"]}}, \"TokenChunker:BlueSkiesLaugh\": {}, \"Tokenizer:KindEyesWatch\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
- "book": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"remove_toc\": true, \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:GrumpyGarlicsBake\": {\"hierarchy\": 5, \"include_heading_content\": true, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"hierarchy\"}, \"Tokenizer:HotDonutsRing\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
- "email": "{\"File\": {}, \"Parser:BirdsFlutterHigh\": {\"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\"]}}, \"TokenChunker:WarmBreadSmells\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:NiceWordsSpoken\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
- "general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
- "laws": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"remove_toc\": true, \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:SpicyKeysKick\": {\"hierarchy\": 2, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"hierarchy\"}, \"Tokenizer:PublicJobsTake\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
- "manual": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"ManualChunker:NineInsectsFind\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:FunnyBalloonsGrin\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
- "one": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"OneChunker:DryDrinksVisit\": {}, \"Tokenizer:FrankWeeksListen\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
- "paper": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"pdf\": {\"enable_multi_column\": true, \"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"TitleChunker:SparklySchoolsTravel\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:GreatCarsWash\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
- "picture": "{\"File\": {}, \"Parser:ViewsCaptureLight\": {\"image\": {\"output_format\": \"json\", \"parse_method\": \"ocr\", \"preprocess\": [\"main_content\"], \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"video\": {\"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"3gp\", \"3gpp\", \"avi\", \"flv\", \"mkv\", \"mov\", \"mp4\", \"mpeg\", \"mpg\", \"webm\", \"wmv\"]}}, \"TokenChunker:BrightColorsGlow\": {}, \"Tokenizer:SharpLensFocus\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
- "presentation": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"slides\": {\"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pptx\", \"ppt\"]}}, \"Tokenizer:TallTreesDance\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"PresentationChunker:HappyHillsGlow\": {}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
+ "audio": "{\"File\": {}, \"Parser:SongsFillAir\": {\"audio\": {\"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"aac\", \"aiff\", \"ape\", \"au\", \"da\", \"flac\", \"midi\", \"mp3\", \"ogg\", \"oggvorbis\", \"realaudio\", \"vqf\", \"wav\", \"wave\", \"wma\"]}}, \"TokenChunker:BlueSkiesLaugh\": {}, \"Tokenizer:KindEyesWatch\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
+ "book": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"remove_toc\": true, \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:GrumpyGarlicsBake\": {\"hierarchy\": 5, \"include_heading_content\": true, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"hierarchy\"}, \"Tokenizer:HotDonutsRing\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
+ "email": "{\"File\": {}, \"Parser:BirdsFlutterHigh\": {\"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\"]}}, \"TokenChunker:WarmBreadSmells\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:NiceWordsSpoken\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
+ "general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
+ "laws": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"remove_toc\": true, \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:SpicyKeysKick\": {\"hierarchy\": 2, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"hierarchy\"}, \"Tokenizer:PublicJobsTake\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
+ "manual": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"ManualChunker:NineInsectsFind\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:FunnyBalloonsGrin\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
+ "one": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"OneChunker:DryDrinksVisit\": {}, \"Tokenizer:FrankWeeksListen\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
+ "paper": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"pdf\": {\"enable_multi_column\": true, \"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"TitleChunker:SparklySchoolsTravel\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:GreatCarsWash\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
+ "picture": "{\"File\": {}, \"Parser:ViewsCaptureLight\": {\"image\": {\"output_format\": \"json\", \"parse_method\": \"ocr\", \"preprocess\": [\"main_content\"], \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"video\": {\"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"3gp\", \"3gpp\", \"avi\", \"flv\", \"mkv\", \"mov\", \"mp4\", \"mpeg\", \"mpg\", \"webm\", \"wmv\"]}}, \"TokenChunker:BrightColorsGlow\": {}, \"Tokenizer:SharpLensFocus\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
+ "presentation": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"slides\": {\"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pptx\", \"ppt\"]}}, \"Tokenizer:TallTreesDance\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"PresentationChunker:HappyHillsGlow\": {}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
"qa": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"Tokenizer:ColdCloudsDream\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"QAChunker:TidyCloudsThink\": {}}",
"table": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}, \"column_mode\": \"auto\", \"column_roles\": {}, \"column_names\": []}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TableChunker:FastFoxesJump\": {}, \"Tokenizer:DeepLakesShine\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}}",
"knowledge_compiler": "{\"File\": {}, \"Compiler:KnownSwiftLions\": {\"compilation_template_group_id\": \"\", \"llm_id\": \"\", \"plan\": false}, \"Parser:HipSignsRhyme\": {\"setups\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}}",
diff --git a/internal/ingestion/task/pipeline_executor_test.go b/internal/ingestion/task/pipeline_executor_test.go
index 80222d12f6..62a50f0bc4 100644
--- a/internal/ingestion/task/pipeline_executor_test.go
+++ b/internal/ingestion/task/pipeline_executor_test.go
@@ -575,3 +575,25 @@ func TestCountOriginalChunkIDs(t *testing.T) {
t.Fatalf("compiler products: got %d, want 2", n)
}
}
+
+func TestBuiltInMetadata_ZeroExtractorFallback(t *testing.T) {
+ // Scenario: User constructed a pipeline with 0 Extractor components,
+ // but dataset parser_config has top-level built_in_metadata.
+ parserConfig := map[string]any{
+ "Parser:HipSignsRhyme": map[string]any{
+ "setups": map[string]any{},
+ },
+ "built_in_metadata": []any{
+ map[string]any{"key": "file_name", "type": "string"},
+ map[string]any{"key": "update_time", "type": "time"},
+ },
+ }
+
+ arr, enabled := builtInMetadataFromParserConfig(parserConfig)
+ if !enabled {
+ t.Errorf("expected builtInMetadataFromParserConfig enabled=true for zero-extractor pipeline with top-level built_in_metadata")
+ }
+ if len(arr) != 2 {
+ t.Errorf("expected 2 built-in metadata items, got %d", len(arr))
+ }
+}
diff --git a/internal/service/component_scoped_parser_config.go b/internal/service/component_scoped_parser_config.go
index 55ca470129..cf78bc993f 100644
--- a/internal/service/component_scoped_parser_config.go
+++ b/internal/service/component_scoped_parser_config.go
@@ -18,8 +18,17 @@ func ApplyComponentScopedParserConfig(
}
enableMetadata := parserConfigTruthy(parserConfig["enable_metadata"])
- metadataFields := mergeMetadataFields(parserConfig)
- hasMetadataConfig := hasMetadataConfigShape(parserConfig)
+ if !enableMetadata {
+ if metaMap, ok := parserConfig["metadata"].(map[string]any); ok {
+ enableMetadata = parserConfigTruthy(metaMap["enabled"])
+ }
+ }
+ if !enableMetadata {
+ enableMetadata = parserConfigTruthy(parserConfig["enabled"])
+ }
+
+ metadataFields := extractFieldList(parserConfig, "fields", "metadata")
+ builtInMetadataFields := extractFieldList(parserConfig, "built_in_metadata", "")
for cpnID, raw := range parserConfig {
params, ok := raw.(map[string]any)
@@ -33,13 +42,46 @@ func ApplyComponentScopedParserConfig(
if value, _ := params["llm_id"].(string); strings.TrimSpace(value) == "" && strings.TrimSpace(llmID) != "" {
params["llm_id"] = llmID
}
- if enableMetadata && len(metadataFields) > 0 {
- params["enable_metadata"] = 1
- params["metadata"] = metadataFields
- } else if hasMetadataConfig {
- params["enable_metadata"] = 0
- params["metadata"] = []any{}
+ delete(params, "enable_metadata")
+
+ hasTopLevelFields := len(metadataFields) > 0 || len(builtInMetadataFields) > 0
+
+ if hasTopLevelFields {
+ params["metadata"] = map[string]any{
+ "enabled": enableMetadata,
+ "metadata": metadataFields,
+ "built_in_metadata": builtInMetadataFields,
+ }
+ } else {
+ nodeCustom := extractFieldList(params, "fields", "metadata")
+ nodeBuiltIn := extractFieldList(params, "built_in_metadata", "")
+ nodeEnabled := false
+
+ if metaMap, ok := params["metadata"].(map[string]any); ok && metaMap["enabled"] != nil {
+ nodeEnabled = parserConfigTruthy(metaMap["enabled"])
+ } else if mcMap, ok := params["metadata_config"].(map[string]any); ok && mcMap["enabled"] != nil {
+ nodeEnabled = parserConfigTruthy(mcMap["enabled"])
+ } else if params["enable_metadata"] != nil {
+ nodeEnabled = parserConfigTruthy(params["enable_metadata"])
+ } else if enableMetadata {
+ nodeEnabled = true
+ }
+
+ if len(nodeCustom) > 0 || len(nodeBuiltIn) > 0 || nodeEnabled {
+ params["metadata"] = map[string]any{
+ "enabled": nodeEnabled,
+ "metadata": nodeCustom,
+ "built_in_metadata": nodeBuiltIn,
+ }
+ } else {
+ params["metadata"] = map[string]any{
+ "enabled": false,
+ "metadata": []any{},
+ "built_in_metadata": []any{},
+ }
+ }
}
+ delete(params, "metadata_config")
case strings.HasPrefix(cpnLower, "compiler:") || strings.HasPrefix(cpnLower, "compiler_"):
if value, _ := params["llm_id"].(string); strings.TrimSpace(value) == "" && strings.TrimSpace(llmID) != "" {
params["llm_id"] = llmID
@@ -51,20 +93,66 @@ func ApplyComponentScopedParserConfig(
}
func mergeMetadataFields(parserConfig entity.JSONMap) []any {
- var out []any
- for _, key := range []string{"metadata", "built_in_metadata"} {
- for _, item := range anySlice(parserConfig[key]) {
- field, ok := item.(map[string]any)
- if !ok {
- continue
+ custom := extractFieldList(parserConfig, "fields", "metadata")
+ builtIn := extractFieldList(parserConfig, "built_in_metadata", "")
+ out := make([]any, 0, len(custom)+len(builtIn))
+ out = append(out, custom...)
+ out = append(out, builtIn...)
+ return out
+}
+
+func extractFieldList(parserConfig entity.JSONMap, primaryKey, fallbackKey string) []any {
+ if parserConfig == nil {
+ return []any{}
+ }
+ if raw, ok := parserConfig[primaryKey]; ok && raw != nil {
+ if slice := extractValidFields(raw); len(slice) > 0 || primaryKey == "fields" {
+ return slice
+ }
+ }
+ if fallbackKey != "" {
+ if raw, ok := parserConfig[fallbackKey]; ok && raw != nil {
+ if slice := extractValidFields(raw); len(slice) > 0 {
+ return slice
}
- name, _ := field["key"].(string)
- if strings.TrimSpace(name) == "" {
- continue
+ }
+ }
+ for _, subKey := range []string{"metadata", "metadata_config"} {
+ if metaMap, ok := parserConfig[subKey].(map[string]any); ok {
+ if raw, ok := metaMap[primaryKey]; ok && raw != nil {
+ if slice := extractValidFields(raw); len(slice) > 0 {
+ return slice
+ }
+ }
+ if fallbackKey != "" {
+ if raw, ok := metaMap[fallbackKey]; ok && raw != nil {
+ if slice := extractValidFields(raw); len(slice) > 0 {
+ return slice
+ }
+ }
}
- out = append(out, field)
}
}
+ return []any{}
+}
+
+func extractValidFields(value any) []any {
+ items := anySlice(value)
+ if items == nil {
+ return []any{}
+ }
+ out := make([]any, 0, len(items))
+ for _, item := range items {
+ field, ok := item.(map[string]any)
+ if !ok {
+ continue
+ }
+ name, _ := field["key"].(string)
+ if strings.TrimSpace(name) == "" {
+ continue
+ }
+ out = append(out, field)
+ }
return out
}
@@ -78,6 +166,27 @@ func anySlice(value any) []any {
out = append(out, item)
}
return out
+ case []map[string]string:
+ out := make([]any, 0, len(typed))
+ for _, item := range typed {
+ m := make(map[string]any, len(item))
+ for k, v := range item {
+ m[k] = v
+ }
+ out = append(out, m)
+ }
+ return out
+ case []MetadataConfigField:
+ out := make([]any, 0, len(typed))
+ for _, item := range typed {
+ out = append(out, map[string]any{
+ "key": item.Key,
+ "type": item.Type,
+ "description": item.Description,
+ "enum": item.Enum,
+ })
+ }
+ return out
default:
return nil
}
@@ -90,11 +199,17 @@ func hasMetadataConfigShape(parserConfig entity.JSONMap) bool {
if _, ok := parserConfig["enable_metadata"]; ok {
return true
}
- for _, key := range []string{"metadata", "built_in_metadata"} {
+ if _, ok := parserConfig["enabled"]; ok {
+ return true
+ }
+ for _, key := range []string{"fields", "metadata", "built_in_metadata"} {
if anySlice(parserConfig[key]) != nil {
return true
}
}
+ if _, ok := parserConfig["metadata"].(map[string]any); ok {
+ return true
+ }
return false
}
diff --git a/internal/service/component_scoped_parser_config_test.go b/internal/service/component_scoped_parser_config_test.go
index bce355d2be..0ffb3a6248 100644
--- a/internal/service/component_scoped_parser_config_test.go
+++ b/internal/service/component_scoped_parser_config_test.go
@@ -29,15 +29,20 @@ func TestApplyComponentScopedParserConfig_SyncsExtractorAndCompiler(t *testing.T
if extractor["llm_id"] != "llm-default" {
t.Fatalf("extractor llm_id = %#v, want llm-default", extractor["llm_id"])
}
- if extractor["enable_metadata"] != 1 {
- t.Fatalf("extractor enable_metadata = %#v, want 1", extractor["enable_metadata"])
+ if _, ok := extractor["enable_metadata"]; ok {
+ t.Fatalf("extractor enable_metadata should not be present, got %#v", extractor["enable_metadata"])
}
- wantFields := []any{
- map[string]any{"key": "author", "type": "string"},
- map[string]any{"key": "document_name", "type": "string"},
+ wantMetadata := map[string]any{
+ "enabled": true,
+ "metadata": []any{
+ map[string]any{"key": "author", "type": "string"},
+ },
+ "built_in_metadata": []any{
+ map[string]any{"key": "document_name", "type": "string"},
+ },
}
- if !reflect.DeepEqual(extractor["metadata"], wantFields) {
- t.Fatalf("extractor metadata = %#v, want %#v", extractor["metadata"], wantFields)
+ if !reflect.DeepEqual(extractor["metadata"], wantMetadata) {
+ t.Fatalf("extractor metadata = %#v, want %#v", extractor["metadata"], wantMetadata)
}
compiler := got["Compiler:KnownSwiftLions"].(map[string]any)
@@ -84,12 +89,17 @@ func TestApplyComponentScopedParserConfig_AcceptsTypedMetadataSlices(t *testing.
got := ApplyComponentScopedParserConfig(parserConfig, "tenant-llm")
extractor := got["Extractor:AutoExtractDefault"].(map[string]any)
- wantFields := []any{
- map[string]interface{}{"key": "author", "type": "string"},
- map[string]interface{}{"key": "document_name", "type": "string"},
+ wantMetadata := map[string]any{
+ "enabled": true,
+ "metadata": []any{
+ map[string]interface{}{"key": "author", "type": "string"},
+ },
+ "built_in_metadata": []any{
+ map[string]interface{}{"key": "document_name", "type": "string"},
+ },
}
- if !reflect.DeepEqual(extractor["metadata"], wantFields) {
- t.Fatalf("extractor metadata = %#v, want %#v", extractor["metadata"], wantFields)
+ if !reflect.DeepEqual(extractor["metadata"], wantMetadata) {
+ t.Fatalf("extractor metadata = %#v, want %#v", extractor["metadata"], wantMetadata)
}
}
@@ -102,8 +112,11 @@ func TestApplyComponentScopedParserConfig_ClearsExtractorMetadataWhenDisabled(t
},
"Extractor:AutoExtractDefault": map[string]any{
"enable_metadata": 1,
- "metadata": []any{
- map[string]any{"key": "stale", "type": "string"},
+ "metadata": map[string]any{
+ "enabled": true,
+ "metadata": []any{
+ map[string]any{"key": "stale", "type": "string"},
+ },
},
},
}
@@ -111,11 +124,18 @@ func TestApplyComponentScopedParserConfig_ClearsExtractorMetadataWhenDisabled(t
got := ApplyComponentScopedParserConfig(parserConfig, "tenant-llm")
extractor := got["Extractor:AutoExtractDefault"].(map[string]any)
- if extractor["enable_metadata"] != 0 {
- t.Fatalf("extractor enable_metadata = %#v, want 0", extractor["enable_metadata"])
+ if _, ok := extractor["enable_metadata"]; ok {
+ t.Fatalf("extractor enable_metadata should not be present, got %#v", extractor["enable_metadata"])
}
- if !reflect.DeepEqual(extractor["metadata"], []any{}) {
- t.Fatalf("extractor metadata = %#v, want empty list", extractor["metadata"])
+ wantMetadata := map[string]any{
+ "enabled": false,
+ "metadata": []any{},
+ "built_in_metadata": []any{
+ map[string]interface{}{"key": "document_name", "type": "string"},
+ },
+ }
+ if !reflect.DeepEqual(extractor["metadata"], wantMetadata) {
+ t.Fatalf("extractor metadata = %#v, want %#v", extractor["metadata"], wantMetadata)
}
}
@@ -124,24 +144,117 @@ func TestApplyComponentScopedParserConfig_DoesNotTreatDocumentMetadataValuesAsSc
"metadata": map[string]interface{}{
"author": "Alice",
},
- "Extractor:AutoExtractDefault": map[string]any{
- "enable_metadata": 1,
- "metadata": []any{
- map[string]any{"key": "author", "type": "string"},
- },
- },
+ "Extractor:AutoExtractDefault": map[string]any{},
}
got := ApplyComponentScopedParserConfig(parserConfig, "tenant-llm")
extractor := got["Extractor:AutoExtractDefault"].(map[string]any)
- if extractor["enable_metadata"] != 1 {
- t.Fatalf("extractor enable_metadata = %#v, want 1", extractor["enable_metadata"])
+ wantMetadata := map[string]any{
+ "enabled": false,
+ "metadata": []any{},
+ "built_in_metadata": []any{},
+ }
+ if !reflect.DeepEqual(extractor["metadata"], wantMetadata) {
+ t.Fatalf("extractor metadata = %#v, want %#v", extractor["metadata"], wantMetadata)
+ }
+}
+
+func TestMergeMetadataFields_MergesFieldsAndBuiltIn(t *testing.T) {
+ parserConfig := entity.JSONMap{
+ "metadata": []any{
+ map[string]any{"key": "author", "type": "string"},
+ },
+ "built_in_metadata": []any{
+ map[string]any{"key": "document_name", "type": "string"},
+ },
}
+
+ got := mergeMetadataFields(parserConfig)
want := []any{
map[string]any{"key": "author", "type": "string"},
+ map[string]any{"key": "document_name", "type": "string"},
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("mergeMetadataFields = %#v, want %#v", got, want)
+ }
+}
+
+func TestApplyComponentScopedParserConfig_PreservesNodeOnlyMetadataOnDatasetSave(t *testing.T) {
+ parserConfig := entity.JSONMap{
+ "Extractor:CanvasNode": map[string]any{
+ "llm_id": "model-1",
+ "metadata": map[string]any{
+ "enabled": true,
+ "metadata": []any{
+ map[string]any{"key": "node_author", "type": "string"},
+ },
+ "built_in_metadata": []any{
+ map[string]any{"key": "node_doc_title", "type": "string"},
+ },
+ },
+ },
+ }
+
+ got := ApplyComponentScopedParserConfig(parserConfig, "tenant-llm")
+ node := got["Extractor:CanvasNode"].(map[string]any)
+
+ meta, ok := node["metadata"].(map[string]any)
+ if !ok {
+ t.Fatalf("expected node metadata map, got %T: %#v", node["metadata"], node["metadata"])
+ }
+ if enabled, _ := meta["enabled"].(bool); !enabled {
+ t.Fatalf("expected node metadata.enabled = true, got false")
+ }
+
+ fields, ok := meta["metadata"].([]any)
+ if !ok || len(fields) != 1 {
+ t.Fatalf("expected 1 custom field preserved, got %#v", meta["metadata"])
+ }
+ if fields[0].(map[string]any)["key"] != "node_author" {
+ t.Fatalf("expected node_author preserved, got %#v", fields[0])
+ }
+ builtIn, ok := meta["built_in_metadata"].([]any)
+ if !ok || len(builtIn) != 1 {
+ t.Fatalf("expected 1 built-in field preserved, got %#v", meta["built_in_metadata"])
+ }
+ if builtIn[0].(map[string]any)["key"] != "node_doc_title" {
+ t.Fatalf("expected node_doc_title preserved, got %#v", builtIn[0])
+ }
+}
+
+func TestApplyComponentScopedParserConfig_PreservesNodeOnlyMetadataConfigShape(t *testing.T) {
+ parserConfig := entity.JSONMap{
+ "enable_metadata": false,
+ "Extractor:CanvasNode": map[string]any{
+ "llm_id": "model-1",
+ "metadata_config": map[string]any{
+ "enabled": true,
+ "metadata": []any{
+ map[string]any{"key": "custom_field", "type": "string"},
+ },
+ },
+ },
+ }
+
+ got := ApplyComponentScopedParserConfig(parserConfig, "tenant-llm")
+ node := got["Extractor:CanvasNode"].(map[string]any)
+
+ if _, ok := node["metadata_config"]; ok {
+ t.Fatalf("metadata_config should be deleted from node, got %#v", node["metadata_config"])
+ }
+ meta, ok := node["metadata"].(map[string]any)
+ if !ok {
+ t.Fatalf("expected node metadata map, got %T: %#v", node["metadata"], node["metadata"])
+ }
+ if enabled, _ := meta["enabled"].(bool); !enabled {
+ t.Fatalf("expected node metadata.enabled = true, got false")
+ }
+ fields, ok := meta["metadata"].([]any)
+ if !ok || len(fields) != 1 {
+ t.Fatalf("expected 1 custom field preserved, got %#v", meta["metadata"])
}
- if !reflect.DeepEqual(extractor["metadata"], want) {
- t.Fatalf("extractor metadata = %#v, want %#v", extractor["metadata"], want)
+ if fields[0].(map[string]any)["key"] != "custom_field" {
+ t.Fatalf("expected custom_field preserved, got %#v", fields[0])
}
}
diff --git a/internal/service/dataset/helpers.go b/internal/service/dataset/helpers.go
index 1dee37048b..28aac5e863 100644
--- a/internal/service/dataset/helpers.go
+++ b/internal/service/dataset/helpers.go
@@ -253,7 +253,7 @@ func preserveDatasetParserConfigMetadata(next, existing entity.JSONMap, incoming
if next == nil {
next = entity.JSONMap{}
}
- for _, key := range []string{"metadata", "built_in_metadata", "enable_metadata"} {
+ for _, key := range []string{"fields", "metadata", "built_in_metadata", "enable_metadata"} {
if incoming != nil {
if value, ok := incoming[key]; ok {
next[key] = value
diff --git a/internal/service/dataset/metadata.go b/internal/service/dataset/metadata.go
index 49b25d0e0e..13c929fee0 100644
--- a/internal/service/dataset/metadata.go
+++ b/internal/service/dataset/metadata.go
@@ -77,8 +77,10 @@ func (d *DatasetService) GetMetadataConfig(ctx context.Context, datasetID, tenan
return nil, common.CodeDataError, fmt.Errorf("user '%s' lacks permission for dataset '%s'", tenantID, datasetID)
}
+ metadata := parserConfigValueOrEmptyList(kb.ParserConfig, "metadata")
+
return map[string]interface{}{
- "metadata": parserConfigValueOrEmptyList(kb.ParserConfig, "metadata"),
+ "metadata": metadata,
"built_in_metadata": parserConfigValueOrEmptyList(kb.ParserConfig, "built_in_metadata"),
}, common.CodeSuccess, nil
}
@@ -122,6 +124,18 @@ func (d *DatasetService) UpdateMetadataConfig(ctx context.Context, datasetID, te
}
parserConfig["metadata"] = metadata
parserConfig["built_in_metadata"] = builtInMetadata
+ // enable_metadata state machine:
+ // - If req.Enabled is explicitly set: use *req.Enabled.
+ // - If nil (uninitialized dataset): auto-enable if metadata fields exist.
+ // - If false (explicitly disabled by user): preserve false even when fields are added.
+ // - If fields are empty: auto-disable to avoid invoking extractor on empty schema.
+ if req.Enabled != nil {
+ parserConfig["enable_metadata"] = *req.Enabled
+ } else if parserConfig["enable_metadata"] == nil {
+ parserConfig["enable_metadata"] = len(metadata) > 0 || len(builtInMetadata) > 0
+ } else if len(metadata) == 0 && len(builtInMetadata) == 0 {
+ parserConfig["enable_metadata"] = false
+ }
if tenant, tenantErr := d.tenantDAO.GetByID(ctx, dao.DB, kb.TenantID); tenantErr == nil && tenant != nil {
parserConfig = service.ApplyComponentScopedParserConfig(
parserConfig,
diff --git a/internal/service/dataset/metadata_config_test.go b/internal/service/dataset/metadata_config_test.go
index ddc0f3f248..e5074c251b 100644
--- a/internal/service/dataset/metadata_config_test.go
+++ b/internal/service/dataset/metadata_config_test.go
@@ -235,9 +235,11 @@ func TestDatasetServiceUpdateMetadataConfigSyncsExtractorSchema(t *testing.T) {
Update("parser_config", entity.JSONMap{
"enable_metadata": false,
"Extractor:AutoExtractDefault": map[string]any{
- "enable_metadata": 1,
- "metadata": []any{
- map[string]any{"key": "stale", "type": "string"},
+ "metadata": map[string]any{
+ "enabled": true,
+ "fields": []any{
+ map[string]any{"key": "stale", "type": "string"},
+ },
},
},
}).Error; err != nil {
@@ -274,10 +276,18 @@ func TestDatasetServiceUpdateMetadataConfigSyncsExtractorSchema(t *testing.T) {
if !ok {
t.Fatalf("expected extractor component params, got %#v", persisted.ParserConfig["Extractor:AutoExtractDefault"])
}
- if got := metadataFlagInt(t, extractor["enable_metadata"]); got != 0 {
- t.Fatalf("extractor enable_metadata = %#v, want 0 when top-level flag stays disabled", extractor["enable_metadata"])
+ if _, ok := extractor["enable_metadata"]; ok {
+ t.Fatalf("extractor enable_metadata should not be present, got %#v", extractor["enable_metadata"])
+ }
+ extractorMeta, ok := extractor["metadata"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("expected modular metadata map in extractor, got %#v", extractor["metadata"])
+ }
+ if enabled, ok := extractorMeta["enabled"].(bool); !ok || enabled {
+ t.Fatalf("extractor metadata.enabled = %#v, want false when top-level flag stays disabled", extractorMeta["enabled"])
}
+ // State 2: When enabled, updating fields updates component with enabled = true
if err := dao.DB.Model(&entity.Knowledgebase{}).
Where("id = ?", "kb-1").
Update("parser_config", entity.JSONMap{
@@ -313,27 +323,100 @@ func TestDatasetServiceUpdateMetadataConfigSyncsExtractorSchema(t *testing.T) {
if !ok {
t.Fatalf("expected extractor component params, got %#v", persisted.ParserConfig["Extractor:AutoExtractDefault"])
}
- if got := metadataFlagInt(t, extractor["enable_metadata"]); got != 1 {
- t.Fatalf("extractor enable_metadata = %#v, want 1", extractor["enable_metadata"])
+ extractorMeta, ok = extractor["metadata"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("expected modular metadata map in extractor, got %#v", extractor["metadata"])
+ }
+ if enabled, ok := extractorMeta["enabled"].(bool); !ok || !enabled {
+ t.Fatalf("extractor metadata.enabled = %#v, want true", extractorMeta["enabled"])
+ }
+ gotFields, ok := extractorMeta["metadata"].([]interface{})
+ if !ok {
+ t.Fatalf("extractor metadata.metadata = %#v, want []interface{}", extractorMeta["metadata"])
+ }
+ if len(gotFields) != 1 {
+ t.Fatalf("extractor metadata.metadata len = %d, want 1 (%#v)", len(gotFields), gotFields)
}
- gotFields, ok := extractor["metadata"].([]interface{})
+ gotBuiltIn, ok := extractorMeta["built_in_metadata"].([]interface{})
if !ok {
- t.Fatalf("extractor metadata = %#v, want []interface{}", extractor["metadata"])
- }
- wantFields := []map[string]interface{}{
- {"key": "author", "type": "string"},
- {"key": "document_name", "type": "string"},
- }
- if len(gotFields) != len(wantFields) {
- t.Fatalf("extractor metadata len = %d, want %d (%#v)", len(gotFields), len(wantFields), gotFields)
- }
- for i, want := range wantFields {
- field, ok := gotFields[i].(map[string]interface{})
- if !ok {
- t.Fatalf("extractor metadata[%d] = %#v, want map[string]interface{}", i, gotFields[i])
- }
- if field["key"] != want["key"] || field["type"] != want["type"] {
- t.Fatalf("extractor metadata[%d] = %#v, want key/type %#v", i, field, want)
- }
+ t.Fatalf("extractor metadata.built_in_metadata = %#v, want []interface{}", extractorMeta["built_in_metadata"])
+ }
+ if len(gotBuiltIn) != 1 {
+ t.Fatalf("extractor metadata.built_in_metadata len = %d, want 1 (%#v)", len(gotBuiltIn), gotBuiltIn)
+ }
+
+ // State 3: Empty fields auto-disables enable_metadata
+ _, code, err = (&DatasetService{
+ kbDAO: dao.NewKnowledgebaseDAO(),
+ tenantDAO: dao.NewTenantDAO(),
+ }).UpdateMetadataConfig(ctx, "kb-1", "tenant-1", &service.MetadataConfigRequest{
+ Metadata: []service.MetadataConfigField{},
+ BuiltInMetadata: []service.MetadataConfigField{},
+ })
+ if err != nil {
+ t.Fatalf("UpdateMetadataConfig with empty fields failed: %v", err)
+ }
+ if code != common.CodeSuccess {
+ t.Fatalf("expected success code, got %d", code)
+ }
+ persisted, err = dao.NewKnowledgebaseDAO().GetByID(ctx, db, "kb-1")
+ if err != nil {
+ t.Fatalf("failed to fetch persisted dataset: %v", err)
+ }
+ if enabled, ok := persisted.ParserConfig["enable_metadata"].(bool); !ok || enabled {
+ t.Fatalf("expected parser_config.enable_metadata == false after emptying fields, got %#v", persisted.ParserConfig["enable_metadata"])
+ }
+}
+
+func TestDatasetServiceGetMetadataConfig(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertCreateDatasetTenant(t, "tenant-1")
+ insertDatasetMetadataConfigKB(t, "kb-1", "tenant-1")
+
+ ctx := t.Context()
+ _, code, err := (&DatasetService{
+ kbDAO: dao.NewKnowledgebaseDAO(),
+ tenantDAO: dao.NewTenantDAO(),
+ }).UpdateMetadataConfig(ctx, "kb-1", "tenant-1", &service.MetadataConfigRequest{
+ Metadata: []service.MetadataConfigField{
+ {Key: "author", Type: "string"},
+ },
+ BuiltInMetadata: []service.MetadataConfigField{
+ {Key: "file_name", Type: "string"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("UpdateMetadataConfig failed: %v", err)
+ }
+ if code != common.CodeSuccess {
+ t.Fatalf("expected success code, got %d", code)
+ }
+
+ result, code, err := (&DatasetService{
+ kbDAO: dao.NewKnowledgebaseDAO(),
+ tenantDAO: dao.NewTenantDAO(),
+ }).GetMetadataConfig(ctx, "kb-1", "tenant-1")
+ if err != nil {
+ t.Fatalf("GetMetadataConfig failed: %v", err)
+ }
+ if code != common.CodeSuccess {
+ t.Fatalf("expected success code, got %d", code)
+ }
+ metadata, ok := result["metadata"].([]interface{})
+ if !ok || len(metadata) != 1 {
+ t.Fatalf("expected 1 metadata item, got %#v", result["metadata"])
+ }
+ field0, ok := metadata[0].(map[string]interface{})
+ if !ok || field0["key"] != "author" {
+ t.Fatalf("expected key author, got %#v", metadata[0])
+ }
+ builtIn, ok := result["built_in_metadata"].([]interface{})
+ if !ok || len(builtIn) != 1 {
+ t.Fatalf("expected 1 built_in_metadata item, got %#v", result["built_in_metadata"])
+ }
+ builtIn0, ok := builtIn[0].(map[string]interface{})
+ if !ok || builtIn0["key"] != "file_name" {
+ t.Fatalf("expected key file_name, got %#v", builtIn[0])
}
}
diff --git a/internal/service/dataset_types.go b/internal/service/dataset_types.go
index 6d61253eba..a1be4a7280 100644
--- a/internal/service/dataset_types.go
+++ b/internal/service/dataset_types.go
@@ -115,6 +115,7 @@ type MetadataConfigField struct {
// MetadataConfigRequest mirrors PUT /datasets/:dataset_id/metadata/config.
type MetadataConfigRequest struct {
+ Enabled *bool `json:"enabled,omitempty"`
Metadata []MetadataConfigField `json:"metadata"`
BuiltInMetadata []MetadataConfigField `json:"built_in_metadata"`
}