RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(ingestion): clamp unconfigured embedding limit; document chunker title parity (comment-only) (#17539)
Jack0cb4039be9c0
diff --git a/internal/ingestion/component/chunker/title.go b/internal/ingestion/component/chunker/title.goindex 40662c341f..7484925a5f 100644--- a/internal/ingestion/component/chunker/title.go+++ b/internal/ingestion/component/chunker/title.go@@ -37,15 +37,21 @@// a non-regex text record whose layout flags it as a// section/title/head and whose text passes not_title is promoted to// fallback_level = len(selected_group) + 1. Strategy (1) —-// PDF-outline detection — still requires deepdoc/parser binary-// access and remains the only parity gap.+// PDF-outline detection — is also ported: newLevelContext+// (title.go) reads the Parser-supplied file.outline via+// outlineFromInputs / resolveOutlineLevels / outlineSimilarity+// (port of common.py:_outline_similarity) before falling back to+// the regex/layout branch, wired into group.go / hierarchy.go+// (Chunker omission 1.5, Gap C closed).//-// - The Go port has NO hardcoded BULLET_PATTERN fallback. Heading-// detection relies entirely on the user-supplied `levels` param-// (which templates carry as comprehensive regex families) paired-// with the layout-hint fallback. A PDF without matching levels-// produces BODY_LEVEL-only records — Python's BULLET_PATTERN-based-// tree_merge / hierarchical_merge would still find structure.+// - The Go port SHIPS a hardcoded BULLET_PATTERN fallback+// (Chunker omission 1.7, Gap C closed): when the regex/layout+// branch yields BODY_LEVEL-only records, resolveTitleLevels+// (title.go) calls bulletsCategory over bulletPatterns (title.go,+// mirroring Python BULLET_PATTERN at rag/nlp/__init__.py:258) to+// recover structure from numbered/bulleted list entries, before+// the layout-title fallback. It never overrides an existing level+// assignment.//// - GROUP-TITLE and HIERARCHY-TITLE are separate Go files// (`group.go`, `hierarchy.go`); they share the resolve_levelsdiff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.goindex cf7c5644cc..8e8318b796 100644--- a/internal/ingestion/component/tokenizer.go+++ b/internal/ingestion/component/tokenizer.go@@ -403,13 +403,14 @@ func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, emtexts := make([]string, 0, len(chunks))pairs := make([]int, 0, len(chunks))for i, ck := range chunks {- txt := concatFields(ck, c.param.Fields)- txt = htmlTableRE.ReplaceAllString(txt, " ")+ raw := concatFields(ck, c.param.Fields)+ txt := htmlTableRE.ReplaceAllString(raw, " ")txt = strings.TrimSpace(txt)+ trunc := truncateForEmbedding(txt, embedder.MaxTokens())if txt == "" {continue}- texts = append(texts, truncateForEmbedding(txt, embedder.MaxTokens()))+ texts = append(texts, trunc)pairs = append(pairs, i)}if len(texts) == 0 {@@ -476,15 +477,35 @@ func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, emreturn chunks, tokenCount, nil}-// truncateForEmbedding truncates text to fit within maxTokens,-// reserving 10 tokens for special/model overhead. Mirrors Python-// tokenizer.py truncation: when maxTokens <= 10, the text is-// truncated to empty+// defaultEmbeddingTokenLimit is the safe fallback used when an embedder reports+// no token limit (maxTokens <= 0). It both prevents empty embedding inputs and+// keeps truncation active for every path instead of passing the full text through.+const defaultEmbeddingTokenLimit = 8192++// truncateForEmbedding keeps the first maxTokens tokens of text so it fits the+// embedding model's limit.+//+// For a positive maxTokens it mirrors Python common/token_utils.py:183-185+// `truncate(string, max_len)` (keep the first max_len tokens).+//+// An unconfigured embedder reports maxTokens <= 0. Rather than mirror Python's+// behaviour of returning "" (which would make the embeddings API reject the whole+// batch with "inputs cannot be empty"), Go clamps the limit to a safe default+// (defaultEmbeddingTokenLimit = 8192). This both prevents empty inputs AND keeps+// truncation active for every path (Builtin and generic) instead of silently+// passing the full, untruncated text when no limit is configured.func truncateForEmbedding(text string, maxTokens int) string {- if maxTokens <= 10 {- return ""+ if maxTokens <= 0 {+ maxTokens = defaultEmbeddingTokenLimit+ }+ // Keep a 10-token safety margin, mirroring Python's embedding path+ // (rag/svr/task_executor.py uses `mdl.max_length - 10`). Only apply it+ // when the limit is large enough; for small limits (<=10) keep the full+ // value so the result stays non-empty instead of collapsing to "".+ if maxTokens > 10 {+ maxTokens -= 10}- return tokenizer.TrimContentToTokenLimit(text, maxTokens-10)+ return tokenizer.TrimContentToTokenLimit(text, maxTokens)}func mergeEmbeddingVectors(titleVec, contentVec []float64, titleWeight float64) ([]float64, error) {diff --git a/internal/ingestion/component/tokenizer_unit_test.go b/internal/ingestion/component/tokenizer_unit_test.goindex 9e3bdcd65b..25dcc1f5f7 100644--- a/internal/ingestion/component/tokenizer_unit_test.go+++ b/internal/ingestion/component/tokenizer_unit_test.go@@ -91,8 +91,8 @@ func (s *stubEmbedder) Encode(ctx context.Context, texts []string) ([]EmbeddingR}// newStubEmbedder returns a stub embedder for instance-level resolver injection.-// maxTokens defaults to 2048 so truncateForEmbedding (Diff 14: maxTokens <= 10 -> "")-// does not empty the content text; tests that exercise truncation set maxTokens explicitly.+// maxTokens defaults to 2048 so truncateForEmbedding truncates to the first+// 2048 tokens; tests that exercise truncation set maxTokens explicitly.func newStubEmbedder(dim int) *stubEmbedder {return &stubEmbedder{dim: dim, maxTokens: 2048}}@@ -376,21 +376,58 @@ func TestIsPhantomChunk(t *testing.T) {}}-// TestTruncateForEmbedding_SmallMaxTokens covers Tokenizer Diff-14: when-// maxTokens <= 10, Go must truncate to empty, matching Python's-// truncation-to-empty behaviour (common/token_utils.py:183-185).+// TestTruncateForEmbedding_SmallMaxTokens covers Tokenizer Diff-14. For any+// positive maxTokens, truncateForEmbedding keeps the first maxTokens tokens+// (non-empty) and the result is strictly shorter than the input.+//+// The unconfigured case (maxTokens <= 0) is covered separately by+// TestTruncateForEmbedding_UnconfiguredClampsToDefault: rather than mirroring+// Python's `truncate` (which returns "" for max_len <= 0 and would make the+// embeddings API reject the batch), Go clamps the limit to a safe default so+// every path still truncates instead of passing the full text through.func TestTruncateForEmbedding_SmallMaxTokens(t *testing.T) {- long := strings.Repeat("a", 100)- if got := truncateForEmbedding(long, 5); got != "" {- t.Errorf("truncateForEmbedding(maxTokens=5) = %q, want %q", got, "")+ // Long enough to produce well over 50 tokens, so 5/10/50 are all clearly+ // below the total and truncation is observable.+ long := strings.Repeat("a", 2000)+ if got := truncateForEmbedding(long, 5); got == "" {+ t.Error("truncateForEmbedding(maxTokens=5) returned empty, want first 5 tokens")}- if got := truncateForEmbedding(long, 10); got != "" {- t.Errorf("truncateForEmbedding(maxTokens=10) = %q, want %q", got, "")+ if got := truncateForEmbedding(long, 10); got == "" {+ t.Error("truncateForEmbedding(maxTokens=10) returned empty, want first 10 tokens")}// Normal path: maxTokens > 10 should truncate (not return empty).if got := truncateForEmbedding(long, 50); got == "" {t.Error("truncateForEmbedding(maxTokens=50) returned empty, want truncated text")}+ if got := truncateForEmbedding(long, 50); len(got) >= len(long) {+ t.Errorf("truncateForEmbedding(maxTokens=50) len = %d, want strictly shorter than %d", len(got), len(long))+ }+}++// TestTruncateForEmbedding_UnconfiguredClampsToDefault is the regression gate+// for the root cause behind embedding truncation: an embedder that reports no+// token limit (maxTokens <= 0) must NOT be passed through verbatim. Before the+// central clamp, the generic model_service branch left maxTokens = 0, which+// silently disabled truncation for the whole generic path. The fix clamps+// maxTokens <= 0 to defaultEmbeddingTokenLimit (8192) inside+// truncateForEmbedding itself, so every caller — Builtin and generic alike —+// keeps truncation active.+//+// For a clearly-over-limit input and maxTokens = 0, the result must be non-empty+// AND strictly shorter than the input: proof that it was truncated to the+// default, not returned verbatim.+func TestTruncateForEmbedding_UnconfiguredClampsToDefault(t *testing.T) {+ // ~10000+ CL100K tokens, comfortably above the 8192 default so truncation+ // is observable.+ long := strings.Repeat("hello world ", 5000)+ got := truncateForEmbedding(long, 0)+ if got == "" {+ t.Fatal("truncateForEmbedding(maxTokens=0) returned empty; clamp must keep truncation active")+ }+ if len(got) >= len(long) {+ t.Errorf("truncateForEmbedding(maxTokens=0) len = %d, want strictly shorter than %d; clamp to default must truncate",+ len(got), len(long))+ }}// TestEmbeddingBatchSizeEnvVar covers Tokenizer Omission-3: the batch size