RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(ingestion): unify extractor prompt placeholder rendering and per-chunk substitution (#18355)
jay777213e38b6cacbc0
diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.goindex 6c1e3cb9de..db76fd8936 100644--- a/internal/ingestion/component/extractor.go+++ b/internal/ingestion/component/extractor.go@@ -667,7 +667,15 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map}if len(in.chunks) == 0 {- ans, callErr := c.callText(timeoutCtx, db, in, "")+ // Render the prompt with an empty chunk map so body placeholders+ // resolve (or fall back to appending nothing) before the LLM call.+ // Without this, a template containing {text} would be forwarded+ // to the model unsubstituted.+ callIn := in+ callIn.systemPrompt, callIn.prompt = renderExtractorPrompts(+ in.systemPrompt, in.prompt, map[string]any{}, "",+ )+ ans, callErr := c.callText(timeoutCtx, db, callIn, "")if callErr != nil {return callErr}@@ -704,19 +712,8 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map// chunk field values, mirroring Python's// string_format at extractor.py:103.callIn := in- // buildExtractorMessages appends chunkText to the user message- // unconditionally. Substitute first; if a content-bearing- // placeholder ({text}/{chunks}/{content_with_weight}) was actually- // replaced, the chunk body is already in the prompt — suppress- // the append to avoid duplication.- var subP, subS bool- callIn.prompt, subP = substituteChunkPlaceholders(in.prompt, ck, text)- callIn.systemPrompt, subS = substituteChunkPlaceholders(in.systemPrompt, ck, text)- callChunkText := text- if subP || subS {- callChunkText = ""- }- ans, callErr := c.callText(timeoutCtx, db, callIn, callChunkText)+ callIn.systemPrompt, callIn.prompt = renderExtractorPrompts(in.systemPrompt, in.prompt, ck, text)+ ans, callErr := c.callText(timeoutCtx, db, callIn, "")if callErr != nil {return fmt.Errorf("chunk %d: %w", i, callErr)}@@ -1164,7 +1161,7 @@ func (c *ExtractorComponent) callRaw(ctx context.Context, db *gorm.DB, in extracif err != nil {return nil, err}- msgs := buildExtractorMessages(in.systemPrompt, in.prompt, chunkText, in.chunks)+ msgs := buildExtractorMessages(in.systemPrompt, in.prompt)fitted, fitErr := fitExtractorMessages(ctx, db, in.llmID, msgs)if fitErr != nil {return nil, fitErr@@ -1549,155 +1546,108 @@ func fitExtractorMessages(ctx context.Context, db *gorm.DB, llmID string, msgs [return fitted, nil}-// buildExtractorMessages assembles system + user messages for-// one extraction call. The user prompt is rendered as-// "<prompt>\n\n<chunkText>" so the python behavior of-// substituting the chunk text into the args dict is preserved-// without invoking a template engine.-//-// Prompt placeholders of the form `{ComponentName:ParamName@chunks}`-// are substituted with the joined text of all upstream chunks-// when chunks is non-empty. The python rag/flow/extractor/extractor.py-// build_existing_prompt path performs the same substitution at-// runtime; the Go port surfaces it as a regex on the prompt-// template so the reference resolves without invoking a template engine.-//-// Substitution is opt-in: when chunks is nil/empty the placeholder-// is left intact so a misconfigured template surfaces as a-// clear pattern rather than silently disappearing.-func buildExtractorMessages(system, prompt, chunkText string, chunks []map[string]any) []eschema.Message {+// buildExtractorMessages assembles system + user messages for one extraction+// call. Prompt rendering (placeholder substitution, chunk-text injection, and+// empty-prompt normalization) is performed upstream by renderExtractorPrompts;+// this function is a pure structural assembler with no rendering logic.+func buildExtractorMessages(system, user string) []eschema.Message {out := make([]eschema.Message, 0, 2)if system != "" {out = append(out, eschema.Message{Role: eschema.System, Content: system})}- user := prompt- if chunkText != "" {- if user != "" {- user += "\n\n"- }- user += chunkText- }- if user == "" {- // An empty prompt + empty chunk is a degenerate call.- // The LLM driver returns an error; we surface that- // unchanged.- user = " "- }- user = substitutePromptPlaceholders(user, chunks)out = append(out, eschema.Message{Role: eschema.User, Content: user})return out}-// substitutePromptPlaceholders replaces `{ComponentName:ParamName@chunks}`-// patterns in the user prompt with the joined text of all upstream-// chunks. The python rag/flow/extractor/extractor.py:build_existing_prompt-// path performs the same substitution at runtime using a Jinja-// template; the Go port keeps the regex form because the LLM-// driver does not require Jinja and the surface is small enough to-// avoid pulling in a template engine.-//-// Pattern grammar:-//-// {CmpName:ParamName@chunks}-//-// The CmpName and ParamName are both matched but ignored — the-// substitute is always "the joined chunk text" today. The-// CmpName/ParamName parsing exists so a future per-component-// substitution can extend the function without breaking the-// existing call sites.-func substitutePromptPlaceholders(prompt string, chunks []map[string]any) string {- if prompt == "" || len(chunks) == 0 {- return prompt- }- // Build the substitution payload once. Each chunk's text is- // joined with a blank line so a downstream LLM sees clear- // chunk boundaries.- var b strings.Builder- for i, ck := range chunks {- t, _ := ck["text"].(string)- if t == "" {- t, _ = ck["content_with_weight"].(string)- }- if t == "" {- continue- }- if i > 0 {- b.WriteString("\n\n")- }- b.WriteString(t)- }- repl := b.String()- if repl == "" {- return prompt- }- return placeholderRE.ReplaceAllString(prompt, repl)-}+// unifiedPlaceholderRE matches plain field placeholders {fieldName} and+// canvas macro placeholders {ComponentName:ParamName@fieldName}.+var unifiedPlaceholderRE = regexp.MustCompile(`\{([A-Za-z0-9_]+(:[A-Za-z0-9_]+)?@[A-Za-z0-9_]+|[A-Za-z_][A-Za-z0-9_]*)\}`)-// placeholderRE matches `{CmpName:ParamName@chunks}` patterns in-// Extractor user prompts. The CMP / Param groups are ignored for-// the @chunks variant but kept so the regex rejects arbitrary-// placeholders (a future per-component substitution extends here).-var placeholderRE = regexp.MustCompile(`\{[A-Za-z0-9_]+:[A-Za-z0-9_]+@chunks\}`)--// simplePlaceholderRE matches simple {field_name} placeholders-// (no colons, no @chunks). Used by substituteChunkPlaceholders to-// replace {text}, {content_with_weight}, etc. with the current-// chunk's field values, mirroring Python's string_format-// (agent/component/base.py:602-609).-var simplePlaceholderRE = regexp.MustCompile(`\{[A-Za-z_][A-Za-z0-9_]*\}`)--// contentPlaceholders are the simple placeholders that resolve to the-// chunk body. Substituting any of them indicates the chunk body is-// already embedded in the prompt — so the automatic append in-// buildExtractorMessages must be suppressed to avoid duplication.-// "text" and "chunks" both map to chunkText (the primary chunk body);-// "content_with_weight" is a chunk field carrying the weighted body.-var contentPlaceholders = map[string]bool{+// bodyPlaceholderAliases lists plain placeholder names that resolve to chunk body content.+var bodyPlaceholderAliases = map[string]bool{"text": true,"chunks": true,"content_with_weight": true,}-// substituteChunkPlaceholders replaces {field_name} placeholders in-// the prompt with values from the current chunk map. The special-// aliases "text" and "chunks" map to chunkText (the current chunk's-// primary text), matching Python's `args[chunks_key] = ck["text"]` at-// extractor.py:102. Unmatched placeholders are left as-is.-//-// The "text" placeholder falls back to chunkText when the chunk has no-// explicit "text" field — e.g. when only content_with_weight is present.-// This makes {text}'s resolved value match the append path's `text`-// variable (which also falls back content_with_weight → text).-//-// Returns the substituted prompt and whether any content-bearing-// placeholder was replaced (i.e. the chunk body is now in the prompt).-func substituteChunkPlaceholders(prompt string, ck map[string]any, chunkText string) (string, bool) {- if prompt == "" || ck == nil {- return prompt, false- }- // Build lookup: chunk fields + "chunks" alias + "text" fallback.- lookup := make(map[string]string, len(ck)+2)- for k, v := range ck {- lookup[k] = fmt.Sprintf("%v", v)- }- if _, has := lookup["text"]; !has {- lookup["text"] = chunkText- }- if _, has := lookup["chunks"]; !has {- lookup["chunks"] = chunkText- }- var substituted bool- out := simplePlaceholderRE.ReplaceAllStringFunc(prompt, func(match string) string {- key := match[1 : len(match)-1] // strip { }- if val, ok := lookup[key]; ok {- if contentPlaceholders[key] {- substituted = true+// isBodyPlaceholder reports whether key represents chunk body content+// (including upstream output references such as @chunks, @text, @markdown).+func isBodyPlaceholder(key string) bool {+ if bodyPlaceholderAliases[key] {+ return true+ }+ return strings.HasSuffix(key, "@chunks") ||+ strings.HasSuffix(key, "@text") ||+ strings.HasSuffix(key, "@markdown")+}++// renderExtractorPrompts performs single-pass rendering of system and user+// prompt templates for one chunk:+// 1. Content placeholders ({text}, {chunks}, {content_with_weight},+// {ComponentName:ParamName@chunks}, etc.) are resolved: first from the chunk+// map, falling back to chunkText. If a non-empty value is resolved,+// bodyInjected is marked true.+// 2. Chunk metadata fields (such as {title}, {author}) are resolved from the+// chunk map.+// 3. Unrecognized placeholders are preserved verbatim.+// 4. If no non-empty chunk body was injected anywhere in the system or user+// prompt, chunkText is automatically appended to the user prompt as a fallback.+func renderExtractorPrompts(sysTemplate, userTemplate string, ck map[string]any, chunkText string) (string, string) {+ var bodyInjected bool++ render := func(tmpl string) string {+ if tmpl == "" {+ return ""+ }+ return unifiedPlaceholderRE.ReplaceAllStringFunc(tmpl, func(match string) string {+ key := match[1 : len(match)-1] // strip { }++ // A. Content placeholders: check chunk map first, fallback to chunkText+ if isBodyPlaceholder(key) {+ var val string+ if raw, ok := ck[key]; ok {+ val = fmt.Sprintf("%v", raw)+ }+ if strings.TrimSpace(val) == "" {+ val = chunkText+ }+ if strings.TrimSpace(val) != "" {+ bodyInjected = true+ }+ return val}- return val++ // B. Chunk metadata fields+ if val, ok := ck[key]; ok {+ return fmt.Sprintf("%v", val)+ }++ // C. Leave unknown placeholders as-is+ return match+ })+ }++ renderedSys := render(sysTemplate)+ renderedUser := render(userTemplate)++ if !bodyInjected && strings.TrimSpace(chunkText) != "" {+ // TrimSpace before testing emptiness is intentional: a user template+ // that renders to pure whitespace (e.g. " ") must not produce a+ // leading "\n\n" separator — the result should be chunkText alone.+ // A plain `if renderedUser != ""` check would fail that invariant.+ trimmed := strings.TrimSpace(renderedUser)+ if trimmed != "" {+ renderedUser = trimmed + "\n\n" + chunkText+ } else {+ renderedUser = chunkText}- return match // leave unknown placeholders as-is- })- return out, substituted+ }++ if strings.TrimSpace(renderedUser) == "" {+ renderedUser = " "+ }++ return renderedSys, renderedUser}// tryParseJSONObject tries to parse s as a JSON object. Returnsdiff --git a/internal/ingestion/component/extractor_test.go b/internal/ingestion/component/extractor_test.goindex 8e036f9015..e4f8ca77e3 100644--- a/internal/ingestion/component/extractor_test.go+++ b/internal/ingestion/component/extractor_test.go@@ -49,10 +49,9 @@ type stubExtractorChatInvoker struct {// as the wrap-error. tests set entries == call count they expect.responses []stubResponse- // lastReq records the most recent call's request for inspection- // (e.g. driver / model name resolved from the llm_id).- lastReq extractorChatRequest- calls atomic.Int32+ // requests records every call in order. Callers read via lastRequest().+ requests []extractorChatRequest+ calls atomic.Int32}// stubResponse couples a Content value and an Err. tests populate@@ -65,7 +64,7 @@ type stubResponse struct {func (s *stubExtractorChatInvoker) Chat(_ context.Context, req extractorChatRequest) (*extractorChatResponse, error) {s.calls.Add(1)s.mu.Lock()- s.lastReq = req+ s.requests = append(s.requests, req)var resp stubResponseif len(s.responses) > 0 {resp = s.responses[0]@@ -78,6 +77,14 @@ func (s *stubExtractorChatInvoker) Chat(_ context.Context, req extractorChatRequreturn &extractorChatResponse{Content: resp.Content}, nil}+// lastRequest returns the most recent recorded request. Callers must hold s.mu.+func (s *stubExtractorChatInvoker) lastRequest() extractorChatRequest {+ if len(s.requests) == 0 {+ return extractorChatRequest{}+ }+ return s.requests[len(s.requests)-1]+}+func (s *stubExtractorChatInvoker) Calls() int32 { return s.calls.Load() }// withStubChatInvoker installs a stub invoker for the duration of@@ -465,8 +472,8 @@ func TestExtractorComponent_Invoke_PerCallLLMIDOverride(t *testing.T) {}stub.mu.Lock()defer stub.mu.Unlock()- if stub.lastReq.ModelName != "override-llm" {- t.Errorf("ModelName = %q, want override-llm", stub.lastReq.ModelName)+ if stub.lastRequest().ModelName != "override-llm" {+ t.Errorf("ModelName = %q, want override-llm", stub.lastRequest().ModelName)}}@@ -488,11 +495,11 @@ func TestExtractorComponent_Invoke_CompositeLLMID(t *testing.T) {}stub.mu.Lock()defer stub.mu.Unlock()- if stub.lastReq.Driver != "openai" {- t.Errorf("Driver = %q, want openai", stub.lastReq.Driver)+ if stub.lastRequest().Driver != "openai" {+ t.Errorf("Driver = %q, want openai", stub.lastRequest().Driver)}- if stub.lastReq.ModelName != "gpt-4o-mini" {- t.Errorf("ModelName = %q, want gpt-4o-mini", stub.lastReq.ModelName)+ if stub.lastRequest().ModelName != "gpt-4o-mini" {+ t.Errorf("ModelName = %q, want gpt-4o-mini", stub.lastRequest().ModelName)}}@@ -1167,11 +1174,11 @@ func TestExtractorComponent_Invoke_TemperatureSet(t *testing.T) {stub.mu.Lock()defer stub.mu.Unlock()- if stub.lastReq.Temperature == nil {+ if stub.lastRequest().Temperature == nil {t.Fatal("Temperature is nil, want 0.2")}- if *stub.lastReq.Temperature != 0.2 {- t.Errorf("Temperature = %v, want 0.2", *stub.lastReq.Temperature)+ if *stub.lastRequest().Temperature != 0.2 {+ t.Errorf("Temperature = %v, want 0.2", *stub.lastRequest().Temperature)}if stub.calls.Load() != 1 {t.Errorf("expected exactly 1 LLM call (keyword), got %d", stub.calls.Load())@@ -1200,8 +1207,8 @@ func TestExtractorComponent_Invoke_FieldNameTemperatureDefault(t *testing.T) {stub.mu.Lock()defer stub.mu.Unlock()- if stub.lastReq.Temperature != nil {- t.Errorf("Temperature = %v, want nil (field extraction uses model default)", *stub.lastReq.Temperature)+ if stub.lastRequest().Temperature != nil {+ t.Errorf("Temperature = %v, want nil (field extraction uses model default)", *stub.lastRequest().Temperature)}}@@ -1488,7 +1495,7 @@ func TestExtractorComponent_Invoke_ContentWithWeightPlaceholder(t *testing.T) {stub.mu.Lock()defer stub.mu.Unlock()var userContent string- for _, msg := range stub.lastReq.Messages {+ for _, msg := range stub.lastRequest().Messages {if msg.Role == eschema.User {userContent = msg.Content}@@ -1530,7 +1537,7 @@ func TestExtractorComponent_Invoke_NonContentPlaceholderKeepsChunkText(t *testinstub.mu.Lock()defer stub.mu.Unlock()var userContent string- for _, msg := range stub.lastReq.Messages {+ for _, msg := range stub.lastRequest().Messages {if msg.Role == eschema.User {userContent = msg.Content}@@ -1570,7 +1577,7 @@ func TestExtractorComponent_Invoke_UnresolvedTextPlaceholderKeepsChunkText(t *testub.mu.Lock()defer stub.mu.Unlock()var userContent string- for _, msg := range stub.lastReq.Messages {+ for _, msg := range stub.lastRequest().Messages {if msg.Role == eschema.User {userContent = msg.Content}@@ -1606,7 +1613,7 @@ func TestExtractorComponent_Invoke_SubstitutesPlaceholders(t *testing.T) {stub.mu.Lock()defer stub.mu.Unlock()var userContent string- for _, msg := range stub.lastReq.Messages {+ for _, msg := range stub.lastRequest().Messages {if msg.Role == eschema.User {userContent = msg.Content}@@ -1648,7 +1655,7 @@ func TestExtractorComponent_Invoke_PlaceholderChunksAlias(t *testing.T) {stub.mu.Lock()defer stub.mu.Unlock()var userContent string- for _, msg := range stub.lastReq.Messages {+ for _, msg := range stub.lastRequest().Messages {if msg.Role == eschema.User {userContent = msg.Content}@@ -1688,7 +1695,7 @@ func TestExtractorComponent_Invoke_AppendsChunkTextWhenNoPlaceholder(t *testing.stub.mu.Lock()defer stub.mu.Unlock()var userContent string- for _, msg := range stub.lastReq.Messages {+ for _, msg := range stub.lastRequest().Messages {if msg.Role == eschema.User {userContent = msg.Content}@@ -1724,7 +1731,7 @@ func TestExtractorComponent_Invoke_SystemPromptPlaceholderSuppressesAppend(t *testub.mu.Lock()defer stub.mu.Unlock()var sysContent, userContent string- for _, msg := range stub.lastReq.Messages {+ for _, msg := range stub.lastRequest().Messages {switch msg.Role {case eschema.System:sysContent = msg.Content@@ -1773,7 +1780,7 @@ func TestExtractorComponent_Invoke_FieldValueContainsPlaceholderSubstring(t *tesstub.mu.Lock()defer stub.mu.Unlock()var userContent string- for _, msg := range stub.lastReq.Messages {+ for _, msg := range stub.lastRequest().Messages {if msg.Role == eschema.User {userContent = msg.Content}@@ -1865,17 +1872,21 @@ func TestExtractorComponent_CallRaw_FitsBeforeInvoke(t *testing.T) {stub := withStubChatInvoker(t, stubResponse{Content: `{"ok": true}`})c := &ExtractorComponent{}+ // callRaw is a pure dispatcher: callers are responsible for pre-rendering+ // the prompt. Inline a large chunk body directly into the user prompt so+ // fitExtractorMessages has real content to trim.+ chunkBody := strings.Repeat("chunk text with lots of tokens. ", 500)_, err := c.callText(t.Context(), nil, extractorInputs{systemPrompt: "extract fields",- prompt: "summarize",+ prompt: "summarize\n\n" + chunkBody,llmID: "test@test",- }, strings.Repeat("chunk text with lots of tokens. ", 500))+ }, "")if err != nil {t.Fatalf("callText: %v", err)}stub.mu.Lock()- req := stub.lastReq+ req := stub.lastRequest()stub.mu.Unlock()if len(req.Messages) == 0 {t.Fatal("invoker was not called")@@ -1931,7 +1942,7 @@ func TestExtractorComponent_CallRaw_CustomContextOverride(t *testing.T) {}stub.mu.Lock()- req := stub.lastReq+ req := stub.lastRequest()stub.mu.Unlock()if len(req.Messages) == 0 {t.Fatal("invoker was not called")@@ -2104,3 +2115,216 @@ func TestExtractorContextLength_NilDBGraceful(t *testing.T) {t.Fatalf("extractorContextLength(nil db, default model) = %d, want 0 (skip fitting)", got)}}++// TestExtractorComponent_Invoke_AtChunksPlaceholderPerChunk verifies that+// when the prompt contains {ComponentName:ParamName@chunks}, each chunk's+// LLM invocation receives ONLY that chunk's text (not all chunks joined together),+// and the chunk text appears exactly once without duplication.+func TestExtractorComponent_Invoke_AtChunksPlaceholderPerChunk(t *testing.T) {+ stub := withStubChatInvoker(t,+ stubResponse{Content: "answer1"},+ stubResponse{Content: "answer2"},+ )++ c := &ExtractorComponent{Param: schema.ExtractorParam{+ FieldName: "summary",+ Prompt: "Summarize: {TokenChunker:BumpyStarsPress@chunks}",+ LLMID: "gpt-4o-mini",+ }}++ _, err := c.Invoke(t.Context(), nil, map[string]any{+ "chunks": []map[string]any{+ {"text": "Chunk One Body"},+ {"text": "Chunk Two Body"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }++ stub.mu.Lock()+ defer stub.mu.Unlock()++ if len(stub.requests) != 2 {+ t.Fatalf("got %d LLM calls, want 2", len(stub.requests))+ }++ // Call 1 must contain Chunk 1 exactly once, and NOT contain Chunk 2+ req1User := stub.requests[0].Messages[len(stub.requests[0].Messages)-1].Content+ if n := strings.Count(req1User, "Chunk One Body"); n != 1 {+ t.Errorf("call 1 chunk text count = %d, want 1; prompt: %q", n, req1User)+ }+ if strings.Contains(req1User, "Chunk Two Body") {+ t.Errorf("call 1 wrongly contains Chunk Two Body; prompt: %q", req1User)+ }++ // Call 2 must contain Chunk 2 exactly once, and NOT contain Chunk 1+ req2User := stub.requests[1].Messages[len(stub.requests[1].Messages)-1].Content+ if n := strings.Count(req2User, "Chunk Two Body"); n != 1 {+ t.Errorf("call 2 chunk text count = %d, want 1; prompt: %q", n, req2User)+ }+ if strings.Contains(req2User, "Chunk One Body") {+ t.Errorf("call 2 wrongly contains Chunk One Body; prompt: %q", req2User)+ }+}++// TestRenderExtractorPrompts_TableDriven covers all prompt rendering cases+// and fallback permutations.+func TestRenderExtractorPrompts_TableDriven(t *testing.T) {+ tests := []struct {+ name string+ sysTemplate string+ userTemplate string+ ck map[string]any+ chunkText string+ wantSys string+ wantUser string+ }{+ {+ name: "text in user",+ sysTemplate: "",+ userTemplate: "Analyze: {text}",+ ck: map[string]any{"text": "body content"},+ chunkText: "body content",+ wantSys: "",+ wantUser: "Analyze: body content",+ },+ {+ name: "text in system only suppresses user append",+ sysTemplate: "Context: {text}",+ userTemplate: "Extract:",+ ck: map[string]any{"text": "body content"},+ chunkText: "body content",+ wantSys: "Context: body content",+ wantUser: "Extract:",+ },+ {+ name: "canvas macro @chunks in user",+ sysTemplate: "",+ userTemplate: "Summarize: {TokenChunker:BumpyStarsPress@chunks}",+ ck: map[string]any{"text": "body content"},+ chunkText: "body content",+ wantSys: "",+ wantUser: "Summarize: body content",+ },+ {+ name: "canvas macro @text in user",+ sysTemplate: "",+ userTemplate: "Parse: {Parser:Doc@text}",+ ck: map[string]any{"text": "body content"},+ chunkText: "body content",+ wantSys: "",+ wantUser: "Parse: body content",+ },+ {+ name: "canvas macro @markdown in user",+ sysTemplate: "",+ userTemplate: "Parse: {Parser:Doc@markdown}",+ ck: map[string]any{"text": "body content"},+ chunkText: "body content",+ wantSys: "",+ wantUser: "Parse: body content",+ },+ {+ name: "metadata only appends chunkText",+ sysTemplate: "",+ userTemplate: "Title: {title}",+ ck: map[string]any{"title": "DocTitle", "text": "body content"},+ chunkText: "body content",+ wantSys: "",+ wantUser: "Title: DocTitle\n\nbody content",+ },+ {+ name: "no placeholder appends chunkText",+ sysTemplate: "",+ userTemplate: "Summarize the text:",+ ck: map[string]any{"text": "body content"},+ chunkText: "body content",+ wantSys: "",+ wantUser: "Summarize the text:\n\nbody content",+ },+ {+ name: "empty chunkText does not append",+ sysTemplate: "",+ userTemplate: "Summarize:",+ ck: map[string]any{},+ chunkText: "",+ wantSys: "",+ wantUser: "Summarize:",+ },+ {+ name: "content_with_weight present in ck",+ sysTemplate: "",+ userTemplate: "Weighted: {content_with_weight}",+ ck: map[string]any{"content_with_weight": "weighted content", "text": "plain content"},+ chunkText: "weighted content",+ wantSys: "",+ wantUser: "Weighted: weighted content",+ },+ {+ name: "content_with_weight absent in ck falls back to chunkText",+ sysTemplate: "",+ userTemplate: "Weighted: {content_with_weight}",+ ck: map[string]any{"text": "plain content"},+ chunkText: "plain content",+ wantSys: "",+ wantUser: "Weighted: plain content",+ },+ {+ name: "whitespace user template trimmed before append",+ sysTemplate: "",+ userTemplate: " ",+ ck: map[string]any{"text": "body content"},+ chunkText: "body content",+ wantSys: "",+ wantUser: "body content",+ },+ {+ name: "empty templates produce single space user turn",+ sysTemplate: "",+ userTemplate: "",+ ck: map[string]any{},+ chunkText: "",+ wantSys: "",+ wantUser: " ",+ },+ {+ // content_with_weight is present in ck but its value is an empty+ // string. isBodyPlaceholder fires, but the resolved value is empty+ // so bodyInjected must NOT be set — the fallback append should+ // still fire and deliver chunkText.+ name: "content_with_weight empty string in ck falls back to chunkText",+ sysTemplate: "",+ userTemplate: "Weighted: {content_with_weight}",+ ck: map[string]any{"content_with_weight": ""},+ chunkText: "fallback body",+ wantSys: "",+ wantUser: "Weighted: fallback body",+ },+ {+ // Both system and user prompts reference body placeholders.+ // The body must appear in each substitution position but must NOT+ // be appended a third time (bodyInjected is shared across both+ // render calls).+ name: "body placeholder in both system and user no extra append",+ sysTemplate: "Context: {text}",+ userTemplate: "Also: {chunks}",+ ck: map[string]any{"text": "the body"},+ chunkText: "the body",+ wantSys: "Context: the body",+ wantUser: "Also: the body",+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ gotSys, gotUser := renderExtractorPrompts(tt.sysTemplate, tt.userTemplate, tt.ck, tt.chunkText)+ if gotSys != tt.wantSys {+ t.Errorf("renderExtractorPrompts() gotSys = %q, want %q", gotSys, tt.wantSys)+ }+ if gotUser != tt.wantUser {+ t.Errorf("renderExtractorPrompts() gotUser = %q, want %q", gotUser, tt.wantUser)+ }+ })+ }+}diff --git a/internal/ingestion/component/slice3_test.go b/internal/ingestion/component/slice3_test.goindex 387e1118d7..4b64be01c4 100644--- a/internal/ingestion/component/slice3_test.go+++ b/internal/ingestion/component/slice3_test.go@@ -25,62 +25,51 @@ import ("testing")-// TestSubstitutePromptPlaceholders_ReplacesAtChunks pins the+// TestRenderExtractorPrompts_ReplacesAtChunks pins the// `{ComponentName:ParamName@chunks}` placeholder substitution.-// The substitute is the joined chunk text.-func TestSubstitutePromptPlaceholders_ReplacesAtChunks(t *testing.T) {+func TestRenderExtractorPrompts_ReplacesAtChunks(t *testing.T) {prompt := "Extract metadata from: {TitleChunker:FlatMiceFix@chunks}"- chunks := []map[string]any{- {"text": "First chunk."},- {"text": "Second chunk."},- }- got := substitutePromptPlaceholders(prompt, chunks)+ ck := map[string]any{"text": "First chunk."}+ _, got := renderExtractorPrompts("", prompt, ck, "First chunk.")if strings.Contains(got, "{TitleChunker:FlatMiceFix@chunks}") {t.Errorf("placeholder not substituted: %q", got)}- if !strings.Contains(got, "First chunk.") || !strings.Contains(got, "Second chunk.") {+ if !strings.Contains(got, "First chunk.") {t.Errorf("substitute missing chunk content: %q", got)}+ if n := strings.Count(got, "First chunk."); n != 1 {+ t.Errorf("chunk text appears %d times, want 1: %q", n, got)+ }}-// TestSubstitutePromptPlaceholders_LeavesPatternWhenNoChunks pins-// the opt-in substitution rule. When chunks is empty the-// placeholder is left intact so a misconfigured template surfaces-// as a clear pattern rather than silently disappearing.-func TestSubstitutePromptPlaceholders_LeavesPatternWhenNoChunks(t *testing.T) {- prompt := "Extract metadata from: {TitleChunker:FlatMiceFix@chunks}"- got := substitutePromptPlaceholders(prompt, nil)+// TestRenderExtractorPrompts_LeavesUnknownPattern pins that unrecognized+// placeholders are preserved verbatim.+func TestRenderExtractorPrompts_LeavesUnknownPattern(t *testing.T) {+ prompt := "Extract metadata from: {unknown_placeholder}"+ _, got := renderExtractorPrompts("", prompt, nil, "")if got != prompt {- t.Errorf("empty chunks: placeholder should be preserved\n got: %q\n want: %q", got, prompt)+ t.Errorf("unknown placeholder: pattern should be preserved\n got: %q\n want: %q", got, prompt)}}-// TestSubstitutePromptPlaceholders_NoPlaceholderInPrompt pins the-// no-op behaviour when the prompt carries no @chunks pattern.-func TestSubstitutePromptPlaceholders_NoPlaceholderInPrompt(t *testing.T) {+// TestRenderExtractorPrompts_NoPlaceholderAppendsChunkText pins the+// fallback append behavior when the prompt carries no body placeholder.+func TestRenderExtractorPrompts_NoPlaceholderAppendsChunkText(t *testing.T) {prompt := "Plain prompt with no substitution."- chunks := []map[string]any{{"text": "x"}}- got := substitutePromptPlaceholders(prompt, chunks)- if got != prompt {- t.Errorf("no-placeholder prompt should be unchanged\n got: %q\n want: %q", got, prompt)+ ck := map[string]any{"text": "my chunk text"}+ _, got := renderExtractorPrompts("", prompt, ck, "my chunk text")+ want := "Plain prompt with no substitution.\n\nmy chunk text"+ if got != want {+ t.Errorf("got %q, want %q", got, want)}}-// TestSubstitutePromptPlaceholders_SkipsEmptyChunkText pins the-// per-chunk text trim. A chunk with no text field does not-// contribute a trailing blank line.-func TestSubstitutePromptPlaceholders_SkipsEmptyChunkText(t *testing.T) {- prompt := "p {TitleChunker:FlatMiceFix@chunks} q"- chunks := []map[string]any{- {"text": ""},- {"text": "actual"},- {},- }- got := substitutePromptPlaceholders(prompt, chunks)- if strings.Contains(got, "{TitleChunker:FlatMiceFix@chunks}") {- t.Errorf("placeholder not substituted: %q", got)- }- if !strings.Contains(got, "actual") {- t.Errorf("chunk text missing: %q", got)+// TestRenderExtractorPrompts_SkipsEmptyChunkText pins that an empty+// chunkText does not append a trailing blank line.+func TestRenderExtractorPrompts_SkipsEmptyChunkText(t *testing.T) {+ prompt := "Plain prompt"+ _, got := renderExtractorPrompts("", prompt, map[string]any{}, "")+ if got != prompt {+ t.Errorf("empty chunkText should not alter prompt\n got: %q\n want: %q", got, prompt)}}