RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(go): merge messagefit into internal/tokenizer (#19225)
jay7772145c3a70c2f60
diff --git a/internal/agent/component/llm.go b/internal/agent/component/llm.goindex 7f5ad882be..f738b25b64 100644--- a/internal/agent/component/llm.go+++ b/internal/agent/component/llm.go@@ -27,9 +27,9 @@ import ("ragflow/internal/agent/component/prompts""ragflow/internal/agent/runtime""ragflow/internal/common"- "ragflow/internal/component/messagefit""ragflow/internal/dao""ragflow/internal/entity/models"+ "ragflow/internal/tokenizer""go.uber.org/zap")@@ -1074,21 +1074,21 @@ func fitMessages(systemPrompt string, msgs []schema.Message, maxLength int) ([]s}}- // Convert to messagefit.Message. Track where each entry's text lives+ // Convert to tokenizer.Message. Track where each entry's text lives// (plain Content or a multi-modal text part) so the fitted text can be// written back to the right field. Entries with no text at all- // (image-only turns) carry an empty Content in messagefit and survive+ // (image-only turns) carry an empty Content for fitting and survive// fitting when kept.type fitSource struct {copiedIdx int // index into copied; -1 for the synthetic system promptmultiIdx int // -1 means the text lives in ContenttextInContent bool // the original message carried text in Content}- all := make([]messagefit.Message, 0, 1+len(copied))+ all := make([]tokenizer.Message, 0, 1+len(copied))sources := make([]fitSource, 0, 1+len(copied))if systemPrompt != "" {- all = append(all, messagefit.Message{Role: "system", Content: systemPrompt})+ all = append(all, tokenizer.Message{Role: "system", Content: systemPrompt})sources = append(sources, fitSource{copiedIdx: -1, multiIdx: 0})}@@ -1114,15 +1114,15 @@ func fitMessages(systemPrompt string, msgs []schema.Message, maxLength int) ([]shadText = true}}- all = append(all, messagefit.Message{Role: string(copied[i].Role), Content: text})+ all = append(all, tokenizer.Message{Role: string(copied[i].Role), Content: text})sources = append(sources, fitSource{copiedIdx: i, multiIdx: multiIdx, textInContent: copied[i].Content != ""})}// Use 97% of effective context as the token budget.budget := contextFitBudget(maxLength)- kept, keptIdx, _ := messagefit.Fit(all, budget)+ kept, keptIdx, _ := tokenizer.Fit(all, budget)- // Convert back to []schema.Message. messagefit.Fit reports exactly which+ // Convert back to []schema.Message. tokenizer.Fit reports exactly which// entries are kept (keptIdx); dropped entries are simply absent, so no// empty-content sentinel is needed and image-only turns are preserved.result := make([]schema.Message, 0, len(kept))diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.goindex c73c9aea35..f724cdae85 100644--- a/internal/ingestion/component/extractor.go+++ b/internal/ingestion/component/extractor.go@@ -50,7 +50,6 @@ import ("ragflow/internal/agent/runtime""ragflow/internal/common"- "ragflow/internal/component/messagefit""ragflow/internal/dao""ragflow/internal/engine/redis""ragflow/internal/entity"@@ -1434,7 +1433,7 @@ func defaultChatModelRef(ctx context.Context, db *gorm.DB, tenantID string) strifunc extractorContextFitBudget(ctxLen int) int {budget := int(float64(ctxLen) * 0.97)if budget < 1 {- // Never hand messagefit a <=0 budget: Fit treats <=0 as the 8192+ // Never hand Fit a <=0 budget: Fit treats <=0 as the 8192// default, which would stop trimming entirely for a tiny context.return 1}@@ -1442,7 +1441,7 @@ func extractorContextFitBudget(ctxLen int) int {}// fitExtractorMessages trims msgs to the chat model's context window using-// the shared messagefit fitter (mirrors Python's message_fit_in), dropping+// the shared tokenizer fitter (mirrors Python's message_fit_in), dropping// entries the fitter removed. It returns a clear error instead of letting a// conversation whose final user turn was trimmed to empty reach the provider:// the proportional trim can do that when the system prompt alone exceeds the@@ -1453,11 +1452,11 @@ func fitExtractorMessages(ctx context.Context, db *gorm.DB, llmID string, msgs [if ctxLen <= 0 {return msgs, nil}- fitMsgs := make([]messagefit.Message, len(msgs))+ fitMsgs := make([]tokenizer.Message, len(msgs))for i := range msgs {- fitMsgs[i] = messagefit.Message{Role: string(msgs[i].Role), Content: msgs[i].Content}+ fitMsgs[i] = tokenizer.Message{Role: string(msgs[i].Role), Content: msgs[i].Content}}- kept, keptIdx, _ := messagefit.Fit(fitMsgs, extractorContextFitBudget(ctxLen))+ kept, keptIdx, _ := tokenizer.Fit(fitMsgs, extractorContextFitBudget(ctxLen))fitted := make([]eschema.Message, 0, len(kept))for j, i := range keptIdx {diff --git a/internal/component/messagefit/messagefit.go b/internal/tokenizer/message_fit.gosimilarity index 81%rename from internal/component/messagefit/messagefit.gorename to internal/tokenizer/message_fit.goindex cccd29b3b7..d2e4fb063b 100644--- a/internal/component/messagefit/messagefit.go+++ b/internal/tokenizer/message_fit.go@@ -6,18 +6,10 @@// copy of the License at http://www.apache.org/licenses/LICENSE-2.0//-// Package messagefit trims a message list so its total token count fits-// within a budget. It mirrors Python's rag/prompts/generator.py:message_fit_in.-//-// The package is shared by the agent canvas LLM component and the ingestion-// Extractor component. Callers convert their message type to []Message, call-// Fit, and send the returned kept messages (at keptIdx into the input).-package messagefit+package tokenizerimport ("slices"-- "ragflow/internal/tokenizer")// Message is the minimal representation the fitter needs. Both the agent's@@ -99,7 +91,7 @@ func Fit(msgs []Message, budget int) (kept []Message, keptIdx []int, count int)// Step 3: trim proportionally.if len(kept) == 1 {- kept[0].Content = tokenizer.TrimContentToTokenLimit(kept[0].Content, budget)+ kept[0].Content = TrimContentToTokenLimit(kept[0].Content, budget)return kept, keptIdx, countTokens(kept)}@@ -116,9 +108,9 @@ func Fit(msgs []Message, budget int) (kept []Message, keptIdx []int, count int)last := &kept[len(kept)-1]ll := 0for i := range sys {- ll += tokenizer.NumTokensFromString(sys[i].Content)+ ll += NumTokensFromString(sys[i].Content)}- ll2 := tokenizer.NumTokensFromString(last.Content)+ ll2 := NumTokensFromString(last.Content)total := ll + ll2if total <= 0 {return kept, keptIdx, 0@@ -128,12 +120,12 @@ func Fit(msgs []Message, budget int) (kept []Message, keptIdx []int, count int)// System dominates: preserve the last message and give the// remaining budget to the system messages.preserved := min(ll2, budget)- last.Content = tokenizer.TrimContentToTokenLimit(last.Content, preserved)+ last.Content = TrimContentToTokenLimit(last.Content, preserved)trimSystems(sys, max(0, budget-preserved))} else {preserved := min(ll, budget)trimSystems(sys, preserved)- last.Content = tokenizer.TrimContentToTokenLimit(last.Content, max(0, budget-preserved))+ last.Content = TrimContentToTokenLimit(last.Content, max(0, budget-preserved))}return kept, keptIdx, countTokens(kept)}@@ -154,7 +146,7 @@ func trimSystems(sys []Message, budget int) {}total := 0for i := range sys {- total += tokenizer.NumTokensFromString(sys[i].Content)+ total += NumTokensFromString(sys[i].Content)}if total <= 0 {return@@ -163,21 +155,21 @@ func trimSystems(sys []Message, budget int) {for i := range sys {limit := remainingif i < len(sys)-1 {- tokens := tokenizer.NumTokensFromString(sys[i].Content)+ tokens := NumTokensFromString(sys[i].Content)limit = int(float64(budget) * float64(tokens) / float64(total))if limit > remaining {limit = remaining}}- sys[i].Content = tokenizer.TrimContentToTokenLimit(sys[i].Content, limit)- remaining -= tokenizer.NumTokensFromString(sys[i].Content)+ sys[i].Content = TrimContentToTokenLimit(sys[i].Content, limit)+ remaining -= NumTokensFromString(sys[i].Content)}}func countTokens(msgs []Message) int {total := 0for i := range msgs {- total += tokenizer.NumTokensFromString(msgs[i].Content)+ total += NumTokensFromString(msgs[i].Content)}return total}diff --git a/internal/component/messagefit/messagefit_test.go b/internal/tokenizer/message_fit_test.gosimilarity index 85%rename from internal/component/messagefit/messagefit_test.gorename to internal/tokenizer/message_fit_test.goindex be41823df4..139b18dbac 100644--- a/internal/component/messagefit/messagefit_test.go+++ b/internal/tokenizer/message_fit_test.go@@ -1,11 +1,9 @@-package messagefit+package tokenizerimport ("slices""strings""testing"-- "ragflow/internal/tokenizer")func TestFit_AllFits(t *testing.T) {@@ -51,7 +49,7 @@ func TestFit_Step2_DropsMiddle(t *testing.T) {{Role: "user", Content: middle},{Role: "user", Content: last},}- budget := tokenizer.NumTokensFromString(sysContent) + tokenizer.NumTokensFromString(last)+ budget := NumTokensFromString(sysContent) + NumTokensFromString(last)kept, keptIdx, count := Fit(msgs, budget)if count == 0 {@@ -71,7 +69,7 @@ func TestFit_ExactBudget(t *testing.T) {{Role: "system", Content: "abc"},{Role: "user", Content: "def"},}- budget := tokenizer.NumTokensFromString("abc") + tokenizer.NumTokensFromString("def")+ budget := NumTokensFromString("abc") + NumTokensFromString("def")kept, keptIdx, _ := Fit(msgs, budget)if len(kept) != 2 || !slices.Equal(keptIdx, []int{0, 1}) {@@ -115,16 +113,16 @@ func TestFit_SystemOnlyMessages(t *testing.T) {if len(kept) != 2 || !slices.Equal(keptIdx, []int{0, 1}) {t.Fatalf("got kept=%+v keptIdx=%v, want both systems", kept, keptIdx)}- total := tokenizer.NumTokensFromString(kept[0].Content) + tokenizer.NumTokensFromString(kept[1].Content)+ total := NumTokensFromString(kept[0].Content) + NumTokensFromString(kept[1].Content)if total > budget {t.Errorf("fitted total %d exceeds budget %d", total, budget)}- fitted0 := tokenizer.NumTokensFromString(kept[0].Content)- fitted1 := tokenizer.NumTokensFromString(kept[1].Content)+ fitted0 := NumTokensFromString(kept[0].Content)+ fitted1 := NumTokensFromString(kept[1].Content)if fitted0 == 0 || fitted1 == 0 {t.Errorf("a system message was emptied by fitting: %+v", kept)}- if fitted0 >= tokenizer.NumTokensFromString(sys1) || fitted1 >= tokenizer.NumTokensFromString(sys2) {+ if fitted0 >= NumTokensFromString(sys1) || fitted1 >= NumTokensFromString(sys2) {t.Errorf("system messages not trimmed: %+v", kept)}}@@ -150,18 +148,18 @@ func TestFit_TrimsAllSystemMessages(t *testing.T) {if kept[2].Content != last {t.Errorf("last user message not preserved: %q", kept[2].Content)}- total := tokenizer.NumTokensFromString(kept[0].Content) +- tokenizer.NumTokensFromString(kept[1].Content) +- tokenizer.NumTokensFromString(kept[2].Content)+ total := NumTokensFromString(kept[0].Content) ++ NumTokensFromString(kept[1].Content) ++ NumTokensFromString(kept[2].Content)if total > budget {t.Errorf("fitted total %d exceeds budget %d", total, budget)}- fitted0 := tokenizer.NumTokensFromString(kept[0].Content)- fitted1 := tokenizer.NumTokensFromString(kept[1].Content)+ fitted0 := NumTokensFromString(kept[0].Content)+ fitted1 := NumTokensFromString(kept[1].Content)if fitted0 == 0 || fitted1 == 0 {t.Errorf("a system message was emptied by fitting: %+v", kept)}- if fitted0 >= tokenizer.NumTokensFromString(sys1) || fitted1 >= tokenizer.NumTokensFromString(sys2) {+ if fitted0 >= NumTokensFromString(sys1) || fitted1 >= NumTokensFromString(sys2) {t.Errorf("system messages not trimmed: %+v", kept)}}@@ -187,10 +185,10 @@ func TestFit_Step3_SystemDominates(t *testing.T) {t.Fatalf("keptIdx = %v, want [0 1]", keptIdx)}// User preserved verbatim; system trimmed.- if tokenizer.NumTokensFromString(kept[1].Content) != tokenizer.NumTokensFromString(userContent) {+ if NumTokensFromString(kept[1].Content) != NumTokensFromString(userContent) {t.Errorf("user message not preserved: %+v", kept)}- if tokenizer.NumTokensFromString(kept[0].Content) >= tokenizer.NumTokensFromString(sysContent) {+ if NumTokensFromString(kept[0].Content) >= NumTokensFromString(sysContent) {t.Errorf("system not trimmed: %+v", kept)}}@@ -216,10 +214,10 @@ func TestFit_Step3_UserDominates(t *testing.T) {t.Fatalf("keptIdx = %v, want [0 1]", keptIdx)}// System preserved verbatim; user trimmed.- if tokenizer.NumTokensFromString(kept[0].Content) != tokenizer.NumTokensFromString(sysContent) {+ if NumTokensFromString(kept[0].Content) != NumTokensFromString(sysContent) {t.Errorf("system message not preserved: %+v", kept)}- if tokenizer.NumTokensFromString(kept[1].Content) >= tokenizer.NumTokensFromString(userContent) {+ if NumTokensFromString(kept[1].Content) >= NumTokensFromString(userContent) {t.Errorf("user not trimmed: %+v", kept)}}@@ -248,7 +246,7 @@ func TestFit_Step3_BudgetFilledByLast(t *testing.T) {if kept[0].Content != "" {t.Errorf("system message not trimmed to empty when the last message fills the budget: %+v", kept)}- if tokenizer.NumTokensFromString(kept[1].Content) > budget {+ if NumTokensFromString(kept[1].Content) > budget {t.Errorf("user message exceeds budget after trim: %+v", kept)}}@@ -264,7 +262,7 @@ func TestFit_SingleMessage(t *testing.T) {if !slices.Equal(keptIdx, []int{0}) {t.Fatalf("keptIdx = %v, want [0]", keptIdx)}- if tokenizer.NumTokensFromString(kept[0].Content) > 100 {+ if NumTokensFromString(kept[0].Content) > 100 {t.Errorf("single message not trimmed: %+v", kept)}}