Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(component): add shared message-fit package (#18091)

jay77721fb08a6c5ee03
diff --git a/internal/component/messagefit/messagefit.go b/internal/component/messagefit/messagefit.go
new file mode 100644
index 0000000000..cccd29b3b7
--- /dev/null
+++ b/internal/component/messagefit/messagefit.go
@@ -0,0 +1,183 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (License); you may not
+// use this file except in compliance with the License. You may obtain a
+// copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+//
+
+// Package messagefit trims a message list so its total token count fits
+// within a budget. It mirrors Python's rag/prompts/generator.py:message_fit_in.
+//
+// The package is shared by the agent canvas LLM component and the ingestion
+// Extractor component. Callers convert their message type to []Message, call
+// Fit, and send the returned kept messages (at keptIdx into the input).
+package messagefit
+
+import (
+ "slices"
+
+ "ragflow/internal/tokenizer"
+)
+
+// Message is the minimal representation the fitter needs. Both the agent's
+// schema.Message and ingestion's eschema.Message convert to this.
+type Message struct {
+ // Role is "system", "user", "assistant", etc. Only "system" receives
+ // special treatment during fitting.
+ Role string
+ // Content is the text content that may be truncated.
+ Content string
+}
+
+// Fit trims msgs so the kept messages fit within budget. budget is the
+// caller-chosen token ceiling for the whole conversation — the agent LLM and
+// ingestion Extractor both pass the chat model's context window
+// (content_length), not the generation cap (max_output).
+//
+// It returns the kept messages in original order (trimmed when necessary),
+// their original indices into msgs, and the kept messages' total token count.
+// msgs itself is never modified, and dropped entries are simply absent from
+// kept/keptIdx — no empty-content sentinel is used. A message that is kept but
+// trimmed to empty (e.g. the system share collapses to 0 when the last message
+// alone fills the budget) is still reported as kept, mirroring Python.
+//
+// Strategy (mirrors Python's message_fit_in, with two deliberate tweaks:
+// an exact budget match counts as fitting, and the system share is spread
+// across every retained system message instead of only the first):
+// 1. If everything fits, return as-is.
+// 2. Keep all system messages + the last non-system message, drop the
+// rest; if that fits, return.
+// 3. If still over, trim proportionally:
+// - System dominates (>80% of tokens) → preserve the last message,
+// give the remaining budget to the system messages.
+// - Otherwise → preserve the system messages, give the remaining
+// budget to the last.
+// - Single message → trim to budget directly.
+//
+// budget <= 0 is treated as 8192 (Python's default).
+func Fit(msgs []Message, budget int) (kept []Message, keptIdx []int, count int) {
+ if budget <= 0 {
+ budget = 8192
+ }
+ if len(msgs) == 0 {
+ return nil, nil, 0
+ }
+
+ // Step 1: everything fits (an exact budget match counts as fitting).
+ if total := countTokens(msgs); total <= budget {
+ kept = slices.Clone(msgs)
+ keptIdx = make([]int, len(msgs))
+ for i := range keptIdx {
+ keptIdx[i] = i
+ }
+ return kept, keptIdx, total
+ }
+
+ // Step 2: keep all system + last non-system.
+ kept = make([]Message, 0, len(msgs))
+ keptIdx = make([]int, 0, len(msgs))
+ lastNonSystem := -1
+ for i := range msgs {
+ if msgs[i].Role == "system" {
+ kept = append(kept, msgs[i])
+ keptIdx = append(keptIdx, i)
+ } else {
+ lastNonSystem = i
+ }
+ }
+ if lastNonSystem >= 0 {
+ kept = append(kept, msgs[lastNonSystem])
+ keptIdx = append(keptIdx, lastNonSystem)
+ }
+ if len(kept) == 0 {
+ return nil, nil, 0
+ }
+ if total := countTokens(kept); total <= budget {
+ return kept, keptIdx, total
+ }
+
+ // Step 3: trim proportionally.
+ if len(kept) == 1 {
+ kept[0].Content = tokenizer.TrimContentToTokenLimit(kept[0].Content, budget)
+ return kept, keptIdx, countTokens(kept)
+ }
+
+ // Only system messages were retained (no non-system message): spread the
+ // whole budget across every retained system message.
+ if lastNonSystem < 0 {
+ trimSystems(kept, budget)
+ return kept, keptIdx, countTokens(kept)
+ }
+
+ // kept[:len(kept)-1] are the retained system messages; the last entry
+ // is the final non-system message.
+ sys := kept[:len(kept)-1]
+ last := &kept[len(kept)-1]
+ ll := 0
+ for i := range sys {
+ ll += tokenizer.NumTokensFromString(sys[i].Content)
+ }
+ ll2 := tokenizer.NumTokensFromString(last.Content)
+ total := ll + ll2
+ if total <= 0 {
+ return kept, keptIdx, 0
+ }
+
+ if float64(ll)/float64(total) > 0.8 {
+ // System dominates: preserve the last message and give the
+ // remaining budget to the system messages.
+ preserved := min(ll2, budget)
+ last.Content = tokenizer.TrimContentToTokenLimit(last.Content, preserved)
+ trimSystems(sys, max(0, budget-preserved))
+ } else {
+ preserved := min(ll, budget)
+ trimSystems(sys, preserved)
+ last.Content = tokenizer.TrimContentToTokenLimit(last.Content, max(0, budget-preserved))
+ }
+ return kept, keptIdx, countTokens(kept)
+}
+
+// trimSystems trims each system message so their combined token count fits
+// within budget. The budget is allocated in proportion to each message's
+// original token count, with the last message taking any remainder so the
+// total never exceeds budget.
+func trimSystems(sys []Message, budget int) {
+ if len(sys) == 0 {
+ return
+ }
+ if budget <= 0 {
+ for i := range sys {
+ sys[i].Content = ""
+ }
+ return
+ }
+ total := 0
+ for i := range sys {
+ total += tokenizer.NumTokensFromString(sys[i].Content)
+ }
+ if total <= 0 {
+ return
+ }
+ remaining := budget
+ for i := range sys {
+ limit := remaining
+ if i < len(sys)-1 {
+ tokens := tokenizer.NumTokensFromString(sys[i].Content)
+ limit = int(float64(budget) * float64(tokens) / float64(total))
+ if limit > remaining {
+ limit = remaining
+ }
+ }
+ sys[i].Content = tokenizer.TrimContentToTokenLimit(sys[i].Content, limit)
+ remaining -= tokenizer.NumTokensFromString(sys[i].Content)
+ }
+}
+
+func countTokens(msgs []Message) int {
+ total := 0
+ for i := range msgs {
+ total += tokenizer.NumTokensFromString(msgs[i].Content)
+ }
+ return total
+}
diff --git a/internal/component/messagefit/messagefit_test.go b/internal/component/messagefit/messagefit_test.go
new file mode 100644
index 0000000000..be41823df4
--- /dev/null
+++ b/internal/component/messagefit/messagefit_test.go
@@ -0,0 +1,297 @@
+package messagefit
+
+import (
+ "slices"
+ "strings"
+ "testing"
+
+ "ragflow/internal/tokenizer"
+)
+
+func TestFit_AllFits(t *testing.T) {
+ msgs := []Message{
+ {Role: "system", Content: "hello"},
+ {Role: "user", Content: "world"},
+ }
+ kept, keptIdx, count := Fit(msgs, 100000)
+ if count == 0 {
+ t.Errorf("Fit returned count 0, want > 0")
+ }
+ if len(kept) != 2 || !slices.Equal(keptIdx, []int{0, 1}) {
+ t.Fatalf("got kept=%+v keptIdx=%v, want both messages", kept, keptIdx)
+ }
+ if kept[0].Content != "hello" || kept[1].Content != "world" {
+ t.Errorf("messages modified when they fit: %+v", kept)
+ }
+}
+
+// TestFit_DoesNotMutateInput verifies the non-mutating contract: Fit never
+// rewrites msgs, so a caller cannot accidentally send emptied entries.
+func TestFit_DoesNotMutateInput(t *testing.T) {
+ orig := []Message{
+ {Role: "system", Content: strings.Repeat("s ", 500)},
+ {Role: "user", Content: strings.Repeat("u ", 500)},
+ {Role: "user", Content: "last"},
+ }
+ msgs := slices.Clone(orig)
+ Fit(msgs, 100)
+ if !slices.Equal(msgs, orig) {
+ t.Fatalf("Fit mutated its input:\n got %+v\nwant %+v", msgs, orig)
+ }
+}
+
+func TestFit_Step2_DropsMiddle(t *testing.T) {
+ // system + last user fit within the budget, but all three together do
+ // not, so Step 2 drops the middle user and keeps system + last intact.
+ sysContent := strings.Repeat("x ", 200)
+ middle := "middle"
+ last := "last"
+ msgs := []Message{
+ {Role: "system", Content: sysContent},
+ {Role: "user", Content: middle},
+ {Role: "user", Content: last},
+ }
+ budget := tokenizer.NumTokensFromString(sysContent) + tokenizer.NumTokensFromString(last)
+
+ kept, keptIdx, count := Fit(msgs, budget)
+ if count == 0 {
+ t.Fatalf("Fit returned count 0, want > 0")
+ }
+ if !slices.Equal(keptIdx, []int{0, 2}) {
+ t.Fatalf("keptIdx = %v, want [0 2] (middle dropped)", keptIdx)
+ }
+ if kept[0].Content != sysContent || kept[1].Content != last {
+ t.Errorf("retained messages modified: %+v", kept)
+ }
+}
+
+func TestFit_ExactBudget(t *testing.T) {
+ // A total exactly equal to the budget counts as fitting.
+ msgs := []Message{
+ {Role: "system", Content: "abc"},
+ {Role: "user", Content: "def"},
+ }
+ budget := tokenizer.NumTokensFromString("abc") + tokenizer.NumTokensFromString("def")
+
+ kept, keptIdx, _ := Fit(msgs, budget)
+ if len(kept) != 2 || !slices.Equal(keptIdx, []int{0, 1}) {
+ t.Fatalf("got kept=%+v keptIdx=%v, want both messages kept", kept, keptIdx)
+ }
+ if kept[0].Content != "abc" || kept[1].Content != "def" {
+ t.Errorf("messages modified at exact budget: %+v", kept)
+ }
+}
+
+// TestFit_NoSystem_KeepsOnlyLast locks the Python-parity behavior: without a
+// system message, Step 2 keeps only the last non-system message.
+func TestFit_NoSystem_KeepsOnlyLast(t *testing.T) {
+ msgs := []Message{
+ {Role: "user", Content: strings.Repeat("a ", 300)},
+ {Role: "assistant", Content: strings.Repeat("b ", 300)},
+ {Role: "user", Content: "last"},
+ }
+ kept, keptIdx, _ := Fit(msgs, 100)
+ if !slices.Equal(keptIdx, []int{2}) {
+ t.Fatalf("keptIdx = %v, want [2] (only the last user kept)", keptIdx)
+ }
+ if len(kept) != 1 || kept[0].Content != "last" {
+ t.Fatalf("kept = %+v, want the last user message", kept)
+ }
+}
+
+func TestFit_SystemOnlyMessages(t *testing.T) {
+ sys1 := strings.Repeat("s ", 800)
+ sys2 := strings.Repeat("t ", 200)
+ msgs := []Message{
+ {Role: "system", Content: sys1},
+ {Role: "system", Content: sys2},
+ }
+ const budget = 500
+
+ kept, keptIdx, count := Fit(msgs, budget)
+ if count == 0 {
+ t.Fatalf("Fit returned count 0, want > 0")
+ }
+ if len(kept) != 2 || !slices.Equal(keptIdx, []int{0, 1}) {
+ t.Fatalf("got kept=%+v keptIdx=%v, want both systems", kept, keptIdx)
+ }
+ total := tokenizer.NumTokensFromString(kept[0].Content) + tokenizer.NumTokensFromString(kept[1].Content)
+ if total > budget {
+ t.Errorf("fitted total %d exceeds budget %d", total, budget)
+ }
+ fitted0 := tokenizer.NumTokensFromString(kept[0].Content)
+ fitted1 := tokenizer.NumTokensFromString(kept[1].Content)
+ if fitted0 == 0 || fitted1 == 0 {
+ t.Errorf("a system message was emptied by fitting: %+v", kept)
+ }
+ if fitted0 >= tokenizer.NumTokensFromString(sys1) || fitted1 >= tokenizer.NumTokensFromString(sys2) {
+ t.Errorf("system messages not trimmed: %+v", kept)
+ }
+}
+
+func TestFit_TrimsAllSystemMessages(t *testing.T) {
+ sys1 := strings.Repeat("s ", 800)
+ sys2 := strings.Repeat("t ", 200)
+ last := "last"
+ msgs := []Message{
+ {Role: "system", Content: sys1},
+ {Role: "system", Content: sys2},
+ {Role: "user", Content: last},
+ }
+ const budget = 500
+
+ kept, keptIdx, count := Fit(msgs, budget)
+ if count == 0 {
+ t.Fatalf("Fit returned count 0, want > 0")
+ }
+ if !slices.Equal(keptIdx, []int{0, 1, 2}) {
+ t.Fatalf("keptIdx = %v, want [0 1 2]", keptIdx)
+ }
+ if kept[2].Content != last {
+ t.Errorf("last user message not preserved: %q", kept[2].Content)
+ }
+ total := tokenizer.NumTokensFromString(kept[0].Content) +
+ tokenizer.NumTokensFromString(kept[1].Content) +
+ tokenizer.NumTokensFromString(kept[2].Content)
+ if total > budget {
+ t.Errorf("fitted total %d exceeds budget %d", total, budget)
+ }
+ fitted0 := tokenizer.NumTokensFromString(kept[0].Content)
+ fitted1 := tokenizer.NumTokensFromString(kept[1].Content)
+ if fitted0 == 0 || fitted1 == 0 {
+ t.Errorf("a system message was emptied by fitting: %+v", kept)
+ }
+ if fitted0 >= tokenizer.NumTokensFromString(sys1) || fitted1 >= tokenizer.NumTokensFromString(sys2) {
+ t.Errorf("system messages not trimmed: %+v", kept)
+ }
+}
+
+func TestFit_Step3_SystemDominates(t *testing.T) {
+ // System takes >80% of tokens → preserve user, trim system.
+ sysContent := strings.Repeat("a ", 800) // dominates
+ userContent := strings.Repeat("b ", 100) // small, fits entirely
+ msgs := []Message{
+ {Role: "system", Content: sysContent},
+ {Role: "user", Content: userContent},
+ }
+ const budget = 500
+
+ kept, keptIdx, count := Fit(msgs, budget)
+ if count == 0 {
+ t.Fatalf("Fit returned count 0, want > 0")
+ }
+ if count > budget {
+ t.Errorf("fitted total %d exceeds budget %d", count, budget)
+ }
+ if !slices.Equal(keptIdx, []int{0, 1}) {
+ t.Fatalf("keptIdx = %v, want [0 1]", keptIdx)
+ }
+ // User preserved verbatim; system trimmed.
+ if tokenizer.NumTokensFromString(kept[1].Content) != tokenizer.NumTokensFromString(userContent) {
+ t.Errorf("user message not preserved: %+v", kept)
+ }
+ if tokenizer.NumTokensFromString(kept[0].Content) >= tokenizer.NumTokensFromString(sysContent) {
+ t.Errorf("system not trimmed: %+v", kept)
+ }
+}
+
+func TestFit_Step3_UserDominates(t *testing.T) {
+ // User takes >20% → preserve system, trim user.
+ sysContent := strings.Repeat("a ", 150) // small, fits entirely
+ userContent := strings.Repeat("b ", 800) // dominates
+ msgs := []Message{
+ {Role: "system", Content: sysContent},
+ {Role: "user", Content: userContent},
+ }
+ const budget = 500
+
+ kept, keptIdx, count := Fit(msgs, budget)
+ if count == 0 {
+ t.Fatalf("Fit returned count 0, want > 0")
+ }
+ if count > budget {
+ t.Errorf("fitted total %d exceeds budget %d", count, budget)
+ }
+ if !slices.Equal(keptIdx, []int{0, 1}) {
+ t.Fatalf("keptIdx = %v, want [0 1]", keptIdx)
+ }
+ // System preserved verbatim; user trimmed.
+ if tokenizer.NumTokensFromString(kept[0].Content) != tokenizer.NumTokensFromString(sysContent) {
+ t.Errorf("system message not preserved: %+v", kept)
+ }
+ if tokenizer.NumTokensFromString(kept[1].Content) >= tokenizer.NumTokensFromString(userContent) {
+ t.Errorf("user not trimmed: %+v", kept)
+ }
+}
+
+// TestFit_Step3_BudgetFilledByLast locks the boundary where the last message
+// alone fills the budget (preserved == budget): the system share collapses to
+// 0, so every retained system message is kept but trimmed to empty — it must
+// NOT be reported as dropped. Python's message_fit_in retains the system entry
+// with content "" in this case too.
+func TestFit_Step3_BudgetFilledByLast(t *testing.T) {
+ sysContent := strings.Repeat("a ", 3000) // dominates (>80% of tokens)
+ userContent := strings.Repeat("b ", 600) // alone exceeds the budget
+ msgs := []Message{
+ {Role: "system", Content: sysContent},
+ {Role: "user", Content: userContent},
+ }
+ const budget = 500
+
+ kept, keptIdx, count := Fit(msgs, budget)
+ if !slices.Equal(keptIdx, []int{0, 1}) {
+ t.Fatalf("keptIdx = %v, want [0 1] (both messages still kept)", keptIdx)
+ }
+ if count > budget {
+ t.Errorf("fitted total %d exceeds budget %d", count, budget)
+ }
+ if kept[0].Content != "" {
+ t.Errorf("system message not trimmed to empty when the last message fills the budget: %+v", kept)
+ }
+ if tokenizer.NumTokensFromString(kept[1].Content) > budget {
+ t.Errorf("user message exceeds budget after trim: %+v", kept)
+ }
+}
+
+func TestFit_SingleMessage(t *testing.T) {
+ msgs := []Message{
+ {Role: "system", Content: strings.Repeat("x ", 1000)},
+ }
+ kept, keptIdx, count := Fit(msgs, 100)
+ if count == 0 {
+ t.Fatalf("Fit returned count 0, want > 0")
+ }
+ if !slices.Equal(keptIdx, []int{0}) {
+ t.Fatalf("keptIdx = %v, want [0]", keptIdx)
+ }
+ if tokenizer.NumTokensFromString(kept[0].Content) > 100 {
+ t.Errorf("single message not trimmed: %+v", kept)
+ }
+}
+
+func TestFit_ZeroBudget(t *testing.T) {
+ // budget <= 0 should use 8192 default and not panic.
+ msgs := []Message{
+ {Role: "system", Content: "hello"},
+ {Role: "user", Content: "world"},
+ }
+ kept, keptIdx, count := Fit(msgs, 0)
+ if count == 0 {
+ t.Fatalf("Fit returned count 0, want > 0")
+ }
+ if len(kept) != 2 || !slices.Equal(keptIdx, []int{0, 1}) {
+ t.Fatalf("got kept=%+v keptIdx=%v, want both messages", kept, keptIdx)
+ }
+ if kept[0].Content != "hello" || kept[1].Content != "world" {
+ t.Errorf("messages modified with default budget: %+v", kept)
+ }
+}
+
+func TestFit_Empty(t *testing.T) {
+ if kept, keptIdx, count := Fit(nil, 1000); kept != nil || keptIdx != nil || count != 0 {
+ t.Errorf("Fit(nil) = %v, %v, %d; want nil, nil, 0", kept, keptIdx, count)
+ }
+ if kept, keptIdx, count := Fit([]Message{}, 1000); len(kept) != 0 || len(keptIdx) != 0 || count != 0 {
+ t.Errorf("Fit(empty) = %v, %v, %d; want 0, 0, 0", kept, keptIdx, count)
+ }
+}