Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(go): merge messagefit into internal/tokenizer (#19225)

jay7772145c3a70c2f60
diff --git a/internal/agent/component/llm.go b/internal/agent/component/llm.go
index 7f5ad882be..f738b25b64 100644
--- a/internal/agent/component/llm.go
+++ b/internal/agent/component/llm.go
@@ -27,9 +27,9 @@ import (
"ragflow/internal/agent/component/prompts"
"ragflow/internal/agent/runtime"
"ragflow/internal/common"
- "ragflow/internal/component/messagefit"
"ragflow/internal/dao"
"ragflow/internal/entity/models"
+ "ragflow/internal/tokenizer"
"go.uber.org/zap"
)
@@ -1074,21 +1074,21 @@ func fitMessages(systemPrompt string, msgs []schema.Message, maxLength int) ([]s
}
}
- // Convert to messagefit.Message. Track where each entry's text lives
+ // Convert to tokenizer.Message. Track where each entry's text lives
// (plain Content or a multi-modal text part) so the fitted text can be
// written back to the right field. Entries with no text at all
- // (image-only turns) carry an empty Content in messagefit and survive
+ // (image-only turns) carry an empty Content for fitting and survive
// fitting when kept.
type fitSource struct {
copiedIdx int // index into copied; -1 for the synthetic system prompt
multiIdx int // -1 means the text lives in Content
textInContent bool // the original message carried text in Content
}
- all := make([]messagefit.Message, 0, 1+len(copied))
+ all := make([]tokenizer.Message, 0, 1+len(copied))
sources := make([]fitSource, 0, 1+len(copied))
if systemPrompt != "" {
- all = append(all, messagefit.Message{Role: "system", Content: systemPrompt})
+ all = append(all, tokenizer.Message{Role: "system", Content: systemPrompt})
sources = append(sources, fitSource{copiedIdx: -1, multiIdx: 0})
}
@@ -1114,15 +1114,15 @@ func fitMessages(systemPrompt string, msgs []schema.Message, maxLength int) ([]s
hadText = true
}
}
- all = append(all, messagefit.Message{Role: string(copied[i].Role), Content: text})
+ all = append(all, tokenizer.Message{Role: string(copied[i].Role), Content: text})
sources = append(sources, fitSource{copiedIdx: i, multiIdx: multiIdx, textInContent: copied[i].Content != ""})
}
// Use 97% of effective context as the token budget.
budget := contextFitBudget(maxLength)
- kept, keptIdx, _ := messagefit.Fit(all, budget)
+ kept, keptIdx, _ := tokenizer.Fit(all, budget)
- // Convert back to []schema.Message. messagefit.Fit reports exactly which
+ // Convert back to []schema.Message. tokenizer.Fit reports exactly which
// entries are kept (keptIdx); dropped entries are simply absent, so no
// empty-content sentinel is needed and image-only turns are preserved.
result := make([]schema.Message, 0, len(kept))
diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.go
index c73c9aea35..f724cdae85 100644
--- a/internal/ingestion/component/extractor.go
+++ b/internal/ingestion/component/extractor.go
@@ -50,7 +50,6 @@ import (
"ragflow/internal/agent/runtime"
"ragflow/internal/common"
- "ragflow/internal/component/messagefit"
"ragflow/internal/dao"
"ragflow/internal/engine/redis"
"ragflow/internal/entity"
@@ -1434,7 +1433,7 @@ func defaultChatModelRef(ctx context.Context, db *gorm.DB, tenantID string) stri
func extractorContextFitBudget(ctxLen int) int {
budget := int(float64(ctxLen) * 0.97)
if budget < 1 {
- // Never hand messagefit a <=0 budget: Fit treats <=0 as the 8192
+ // Never hand Fit a <=0 budget: Fit treats <=0 as the 8192
// default, which would stop trimming entirely for a tiny context.
return 1
}
@@ -1442,7 +1441,7 @@ func extractorContextFitBudget(ctxLen int) int {
}
// fitExtractorMessages trims msgs to the chat model's context window using
-// the shared messagefit fitter (mirrors Python's message_fit_in), dropping
+// the shared tokenizer fitter (mirrors Python's message_fit_in), dropping
// entries the fitter removed. It returns a clear error instead of letting a
// conversation whose final user turn was trimmed to empty reach the provider:
// the proportional trim can do that when the system prompt alone exceeds the
@@ -1453,11 +1452,11 @@ func fitExtractorMessages(ctx context.Context, db *gorm.DB, llmID string, msgs [
if ctxLen <= 0 {
return msgs, nil
}
- fitMsgs := make([]messagefit.Message, len(msgs))
+ fitMsgs := make([]tokenizer.Message, len(msgs))
for i := range msgs {
- fitMsgs[i] = messagefit.Message{Role: string(msgs[i].Role), Content: msgs[i].Content}
+ fitMsgs[i] = tokenizer.Message{Role: string(msgs[i].Role), Content: msgs[i].Content}
}
- kept, keptIdx, _ := messagefit.Fit(fitMsgs, extractorContextFitBudget(ctxLen))
+ kept, keptIdx, _ := tokenizer.Fit(fitMsgs, extractorContextFitBudget(ctxLen))
fitted := make([]eschema.Message, 0, len(kept))
for j, i := range keptIdx {
diff --git a/internal/component/messagefit/messagefit.go b/internal/tokenizer/message_fit.go
similarity index 81%
rename from internal/component/messagefit/messagefit.go
rename to internal/tokenizer/message_fit.go
index cccd29b3b7..d2e4fb063b 100644
--- a/internal/component/messagefit/messagefit.go
+++ b/internal/tokenizer/message_fit.go
@@ -6,18 +6,10 @@
// copy of the License at http://www.apache.org/licenses/LICENSE-2.0
//
-// Package messagefit trims a message list so its total token count fits
-// within a budget. It mirrors Python's rag/prompts/generator.py:message_fit_in.
-//
-// The package is shared by the agent canvas LLM component and the ingestion
-// Extractor component. Callers convert their message type to []Message, call
-// Fit, and send the returned kept messages (at keptIdx into the input).
-package messagefit
+package tokenizer
import (
"slices"
-
- "ragflow/internal/tokenizer"
)
// Message is the minimal representation the fitter needs. Both the agent's
@@ -99,7 +91,7 @@ func Fit(msgs []Message, budget int) (kept []Message, keptIdx []int, count int)
// Step 3: trim proportionally.
if len(kept) == 1 {
- kept[0].Content = tokenizer.TrimContentToTokenLimit(kept[0].Content, budget)
+ kept[0].Content = TrimContentToTokenLimit(kept[0].Content, budget)
return kept, keptIdx, countTokens(kept)
}
@@ -116,9 +108,9 @@ func Fit(msgs []Message, budget int) (kept []Message, keptIdx []int, count int)
last := &kept[len(kept)-1]
ll := 0
for i := range sys {
- ll += tokenizer.NumTokensFromString(sys[i].Content)
+ ll += NumTokensFromString(sys[i].Content)
}
- ll2 := tokenizer.NumTokensFromString(last.Content)
+ ll2 := NumTokensFromString(last.Content)
total := ll + ll2
if total <= 0 {
return kept, keptIdx, 0
@@ -128,12 +120,12 @@ func Fit(msgs []Message, budget int) (kept []Message, keptIdx []int, count int)
// System dominates: preserve the last message and give the
// remaining budget to the system messages.
preserved := min(ll2, budget)
- last.Content = tokenizer.TrimContentToTokenLimit(last.Content, preserved)
+ last.Content = TrimContentToTokenLimit(last.Content, preserved)
trimSystems(sys, max(0, budget-preserved))
} else {
preserved := min(ll, budget)
trimSystems(sys, preserved)
- last.Content = tokenizer.TrimContentToTokenLimit(last.Content, max(0, budget-preserved))
+ last.Content = TrimContentToTokenLimit(last.Content, max(0, budget-preserved))
}
return kept, keptIdx, countTokens(kept)
}
@@ -154,7 +146,7 @@ func trimSystems(sys []Message, budget int) {
}
total := 0
for i := range sys {
- total += tokenizer.NumTokensFromString(sys[i].Content)
+ total += NumTokensFromString(sys[i].Content)
}
if total <= 0 {
return
@@ -163,21 +155,21 @@ func trimSystems(sys []Message, budget int) {
for i := range sys {
limit := remaining
if i < len(sys)-1 {
- tokens := tokenizer.NumTokensFromString(sys[i].Content)
+ tokens := NumTokensFromString(sys[i].Content)
limit = int(float64(budget) * float64(tokens) / float64(total))
if limit > remaining {
limit = remaining
}
}
- sys[i].Content = tokenizer.TrimContentToTokenLimit(sys[i].Content, limit)
- remaining -= tokenizer.NumTokensFromString(sys[i].Content)
+ sys[i].Content = TrimContentToTokenLimit(sys[i].Content, limit)
+ remaining -= NumTokensFromString(sys[i].Content)
}
}
func countTokens(msgs []Message) int {
total := 0
for i := range msgs {
- total += tokenizer.NumTokensFromString(msgs[i].Content)
+ total += NumTokensFromString(msgs[i].Content)
}
return total
}
diff --git a/internal/component/messagefit/messagefit_test.go b/internal/tokenizer/message_fit_test.go
similarity index 85%
rename from internal/component/messagefit/messagefit_test.go
rename to internal/tokenizer/message_fit_test.go
index be41823df4..139b18dbac 100644
--- a/internal/component/messagefit/messagefit_test.go
+++ b/internal/tokenizer/message_fit_test.go
@@ -1,11 +1,9 @@
-package messagefit
+package tokenizer
import (
"slices"
"strings"
"testing"
-
- "ragflow/internal/tokenizer"
)
func TestFit_AllFits(t *testing.T) {
@@ -51,7 +49,7 @@ func TestFit_Step2_DropsMiddle(t *testing.T) {
{Role: "user", Content: middle},
{Role: "user", Content: last},
}
- budget := tokenizer.NumTokensFromString(sysContent) + tokenizer.NumTokensFromString(last)
+ budget := NumTokensFromString(sysContent) + NumTokensFromString(last)
kept, keptIdx, count := Fit(msgs, budget)
if count == 0 {
@@ -71,7 +69,7 @@ func TestFit_ExactBudget(t *testing.T) {
{Role: "system", Content: "abc"},
{Role: "user", Content: "def"},
}
- budget := tokenizer.NumTokensFromString("abc") + tokenizer.NumTokensFromString("def")
+ budget := NumTokensFromString("abc") + NumTokensFromString("def")
kept, keptIdx, _ := Fit(msgs, budget)
if len(kept) != 2 || !slices.Equal(keptIdx, []int{0, 1}) {
@@ -115,16 +113,16 @@ func TestFit_SystemOnlyMessages(t *testing.T) {
if len(kept) != 2 || !slices.Equal(keptIdx, []int{0, 1}) {
t.Fatalf("got kept=%+v keptIdx=%v, want both systems", kept, keptIdx)
}
- total := tokenizer.NumTokensFromString(kept[0].Content) + tokenizer.NumTokensFromString(kept[1].Content)
+ total := NumTokensFromString(kept[0].Content) + NumTokensFromString(kept[1].Content)
if total > budget {
t.Errorf("fitted total %d exceeds budget %d", total, budget)
}
- fitted0 := tokenizer.NumTokensFromString(kept[0].Content)
- fitted1 := tokenizer.NumTokensFromString(kept[1].Content)
+ fitted0 := NumTokensFromString(kept[0].Content)
+ fitted1 := NumTokensFromString(kept[1].Content)
if fitted0 == 0 || fitted1 == 0 {
t.Errorf("a system message was emptied by fitting: %+v", kept)
}
- if fitted0 >= tokenizer.NumTokensFromString(sys1) || fitted1 >= tokenizer.NumTokensFromString(sys2) {
+ if fitted0 >= NumTokensFromString(sys1) || fitted1 >= NumTokensFromString(sys2) {
t.Errorf("system messages not trimmed: %+v", kept)
}
}
@@ -150,18 +148,18 @@ func TestFit_TrimsAllSystemMessages(t *testing.T) {
if kept[2].Content != last {
t.Errorf("last user message not preserved: %q", kept[2].Content)
}
- total := tokenizer.NumTokensFromString(kept[0].Content) +
- tokenizer.NumTokensFromString(kept[1].Content) +
- tokenizer.NumTokensFromString(kept[2].Content)
+ total := NumTokensFromString(kept[0].Content) +
+ NumTokensFromString(kept[1].Content) +
+ NumTokensFromString(kept[2].Content)
if total > budget {
t.Errorf("fitted total %d exceeds budget %d", total, budget)
}
- fitted0 := tokenizer.NumTokensFromString(kept[0].Content)
- fitted1 := tokenizer.NumTokensFromString(kept[1].Content)
+ fitted0 := NumTokensFromString(kept[0].Content)
+ fitted1 := NumTokensFromString(kept[1].Content)
if fitted0 == 0 || fitted1 == 0 {
t.Errorf("a system message was emptied by fitting: %+v", kept)
}
- if fitted0 >= tokenizer.NumTokensFromString(sys1) || fitted1 >= tokenizer.NumTokensFromString(sys2) {
+ if fitted0 >= NumTokensFromString(sys1) || fitted1 >= NumTokensFromString(sys2) {
t.Errorf("system messages not trimmed: %+v", kept)
}
}
@@ -187,10 +185,10 @@ func TestFit_Step3_SystemDominates(t *testing.T) {
t.Fatalf("keptIdx = %v, want [0 1]", keptIdx)
}
// User preserved verbatim; system trimmed.
- if tokenizer.NumTokensFromString(kept[1].Content) != tokenizer.NumTokensFromString(userContent) {
+ if NumTokensFromString(kept[1].Content) != NumTokensFromString(userContent) {
t.Errorf("user message not preserved: %+v", kept)
}
- if tokenizer.NumTokensFromString(kept[0].Content) >= tokenizer.NumTokensFromString(sysContent) {
+ if NumTokensFromString(kept[0].Content) >= NumTokensFromString(sysContent) {
t.Errorf("system not trimmed: %+v", kept)
}
}
@@ -216,10 +214,10 @@ func TestFit_Step3_UserDominates(t *testing.T) {
t.Fatalf("keptIdx = %v, want [0 1]", keptIdx)
}
// System preserved verbatim; user trimmed.
- if tokenizer.NumTokensFromString(kept[0].Content) != tokenizer.NumTokensFromString(sysContent) {
+ if NumTokensFromString(kept[0].Content) != NumTokensFromString(sysContent) {
t.Errorf("system message not preserved: %+v", kept)
}
- if tokenizer.NumTokensFromString(kept[1].Content) >= tokenizer.NumTokensFromString(userContent) {
+ if NumTokensFromString(kept[1].Content) >= NumTokensFromString(userContent) {
t.Errorf("user not trimmed: %+v", kept)
}
}
@@ -248,7 +246,7 @@ func TestFit_Step3_BudgetFilledByLast(t *testing.T) {
if kept[0].Content != "" {
t.Errorf("system message not trimmed to empty when the last message fills the budget: %+v", kept)
}
- if tokenizer.NumTokensFromString(kept[1].Content) > budget {
+ if NumTokensFromString(kept[1].Content) > budget {
t.Errorf("user message exceeds budget after trim: %+v", kept)
}
}
@@ -264,7 +262,7 @@ func TestFit_SingleMessage(t *testing.T) {
if !slices.Equal(keptIdx, []int{0}) {
t.Fatalf("keptIdx = %v, want [0]", keptIdx)
}
- if tokenizer.NumTokensFromString(kept[0].Content) > 100 {
+ if NumTokensFromString(kept[0].Content) > 100 {
t.Errorf("single message not trimmed: %+v", kept)
}
}