Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(ingestion): port Python's table-tag strip to the tokenizer embedding path (#21156)

xugangqiangf2b9d65ef1ab
diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.go
index 9d6de83931..525eea0efb 100644
--- a/internal/ingestion/component/tokenizer.go
+++ b/internal/ingestion/component/tokenizer.go
@@ -188,6 +188,23 @@ func resolveEmbedderMaxTokens(embedder Embedder) int {
return embedder.MaxTokens()
}
+// tableStructureTagRe mirrors the Python tokenizer's cleaned_txt step
+// (rag/flow/tokenizer/tokenizer.py:86): table-structure tags are replaced
+// with a space before the emptiness check and before the text is embedded,
+// so a chunk that is nothing but an empty table carries no embeddable
+// content. The attribute group is deliberately bounded to 12 non-tag
+// characters, exactly like the Python pattern: a tag with longer attributes
+// survives the strip. Do not "fix" the bound — it is parity, not a bug.
+var tableStructureTagRe = regexp.MustCompile(`</?(table|td|caption|tr|th)( [^<>]{0,12})?>`)
+
+// cleanEmbedText applies the table-structure strip and trims, mirroring
+// Python's cleaned_txt. The stripped text is both the emptiness check and
+// what gets embedded — Python appends cleaned_txt to the embedding batch
+// (tokenizer.py:89), not the raw chunk.
+func cleanEmbedText(raw string) string {
+ return strings.TrimSpace(tableStructureTagRe.ReplaceAllString(raw, " "))
+}
+
// EmbedderResolver resolves the embedder and its dataset-bound embedding-model
// id for one tokenizer invocation. The resolver derives the model exclusively
// from the knowledgebase's configured embd_id (see internal/ingestion/task/
@@ -470,7 +487,10 @@ func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, na
cacheHits := 0
for i, ck := range chunks {
raw := concatFields(ck, c.param.Fields)
- txt := strings.TrimSpace(raw)
+ // Python parity (tokenizer.py:86-89): strip table-structure tags
+ // before the emptiness check and embed the stripped text, so an
+ // empty-table chunk contributes nothing to the batch.
+ txt := cleanEmbedText(raw)
if txt == "" {
continue
}
@@ -498,11 +518,12 @@ func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, na
truncs = append(truncs, trunc)
pairs = append(pairs, i)
}
- if len(texts) == 0 {
- // Nothing to embed — but cache hits may still need to be written through.
- if store == nil {
- return chunks, 0, nil
- }
+ if len(texts) == 0 && cacheHits == 0 {
+ // Nothing to embed and nothing to serve from cache: return before the
+ // title encode, exactly like Python's `if not texts: return` — an
+ // all-skipped batch must not burn a title embedding. (When the cache
+ // serves every chunk the flow continues so those vectors attach.)
+ return chunks, 0, nil
}
trimmedName := strings.TrimSpace(name)
@@ -1070,12 +1091,11 @@ func requiresFullTextTokens(ck schema.ChunkDoc) bool {
return strings.TrimSpace(ck.Summary) != "" || strings.TrimSpace(schema.ContextualText(ck)) != ""
}
+// requiresEmbeddingVector uses the same cleaning as the embed path
+// (cleanEmbedText): a chunk whose cleaned text is empty was skipped from the
+// embedding batch and therefore legitimately carries no vector.
func requiresEmbeddingVector(ck schema.ChunkDoc, fields []string) bool {
- return strings.TrimSpace(cleanEmbeddingText(concatFields(ck, fields))) != ""
-}
-
-func cleanEmbeddingText(text string) string {
- return strings.TrimSpace(text)
+ return cleanEmbedText(concatFields(ck, fields)) != ""
}
func hasEmbeddingVector(ck schema.ChunkDoc) bool {
diff --git a/internal/ingestion/component/tokenizer_clean_embed_text_test.go b/internal/ingestion/component/tokenizer_clean_embed_text_test.go
new file mode 100644
index 0000000000..cea1038750
--- /dev/null
+++ b/internal/ingestion/component/tokenizer_clean_embed_text_test.go
@@ -0,0 +1,154 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package component
+
+import (
+ "context"
+ "strings"
+ "testing"
+
+ "ragflow/internal/ingestion/component/schema"
+)
+
+// TestCleanEmbedText pins the table-structure strip that mirrors Python's
+// cleaned_txt (rag/flow/tokenizer/tokenizer.py:86). The contract is a direct
+// regex port, quirks included: case-sensitive tag names, and an attribute
+// group bounded to 12 non-tag characters.
+func TestCleanEmbedText(t *testing.T) {
+ cases := []struct {
+ name string
+ in string
+ want string
+ }{
+ {"empty", "", ""},
+ {"whitespace only", " \n\t ", ""},
+ {"plain text unchanged", "just words", "just words"},
+ {"text keeps inner whitespace", "a b\nc", "a b\nc"},
+
+ {"empty table", "<table></table>", ""},
+ {"empty table rows cells", "<table><tr><td></td></tr></table>", ""},
+ {"whitespace-only table", "<table> </table>", ""},
+ {"empty caption", "<caption></caption>", ""},
+ {"empty th", "<th></th>", ""},
+
+ {"table with text", "<table><tr><td>hello</td></tr></table>", "hello"},
+ // Each replaced tag contributes one space; Python's re.sub does not
+ // collapse runs of spaces, and neither does this port.
+ {"text around table", "before<table><tr><td>x</td></tr></table>after", "before x after"},
+ {"nested tables", "<table><tr><td><table><tr><td>deep</td></tr></table></td></tr></table>", "deep"},
+ {"all tag kinds", "<table><caption>cap</caption><tr><th>h</th><td>d</td></tr></table>", "cap h d"},
+
+ {"closing tags alone", "</table></td>", ""},
+ {"attributes within bound", `<td colspan="2">x`, "x"},
+ {"attribute bound exactly twelve", `<td class="abc">x`, "x"},
+ {"attribute over bound survives", `<td class="abcde">x`, `<td class="abcde">x`},
+ {"long attribute opening survives closing does not", `<table class="a-longer-class-name">cell</table>`, `<table class="a-longer-class-name">cell`},
+
+ {"uppercase tags are not stripped (case-sensitive parity)", "<TABLE><TR><TD>x</TD></TR></TABLE>", "<TABLE><TR><TD>x</TD></TR></TABLE>"},
+ {"mixed case tag not stripped", "<Table>x</Table>", "<Table>x</Table>"},
+
+ {"unclosed tag untouched", "<table", "<table"},
+ {"angle bracket soup untouched", "a < b > c", "a < b > c"},
+ {"unicode around tags", "中文<table><tr><td>内容</td></tr></table>尾", "中文 内容 尾"},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ if got := cleanEmbedText(c.in); got != c.want {
+ t.Fatalf("cleanEmbedText(%q) = %q, want %q", c.in, got, c.want)
+ }
+ })
+ }
+}
+
+// TestCleanEmbedTextStripsEveryMatch guards the global (not first-match)
+// substitution: a long text with many table tags must have all of them
+// replaced, not just the first.
+func TestCleanEmbedTextStripsEveryMatch(t *testing.T) {
+ in := "<td>a</td><td>b</td><td>c</td>"
+ got := cleanEmbedText(in)
+ if strings.Contains(got, "<td>") || strings.Contains(got, "</td>") {
+ t.Fatalf("cleanEmbedText(%q) = %q, want every tag replaced", in, got)
+ }
+ if got != "a b c" {
+ t.Fatalf("cleanEmbedText(%q) = %q, want %q", in, got, "a b c")
+ }
+}
+
+// TestEmbedChunksAllSkippedWithCacheStoreSkipsTitleEncode: when every chunk's
+// cleaned text is empty and no cache entry can serve them, nothing may be
+// embedded — not even the title. Python returns before the title encode when
+// the batch is empty (tokenizer.py:92-93); a configured cache store does not
+// change that, it only matters when it actually serves a vector.
+func TestEmbedChunksAllSkippedWithCacheStoreSkipsTitleEncode(t *testing.T) {
+ comp, stub := withStubEmbedder(t, 4)
+ comp.param.Fields = []string{"text"}
+ chunks := []schema.ChunkDoc{
+ chunkWithID("c1", "<table><tr><td></td></tr></table>"),
+ chunkWithID("c2", " "),
+ }
+
+ out, consumed, err := comp.embedChunks(context.Background(), "tenant", "kb", "doc.pdf", chunks, newMemCacheStore())
+ if err != nil {
+ t.Fatalf("embedChunks: %v", err)
+ }
+ if got := stub.calls.Load(); got != 0 {
+ t.Fatalf("embedder calls = %d, want 0 (an all-skipped batch embeds nothing, title included)", got)
+ }
+ if consumed != 0 {
+ t.Fatalf("embedding_token_consumption = %d, want 0", consumed)
+ }
+ for i, ck := range out {
+ if hasEmbeddingVector(ck) {
+ t.Errorf("chunk[%d] unexpectedly carries a vector", i)
+ }
+ }
+}
+
+// TestEmbedChunksCacheHitOnlyBatchStillAttachesVectors guards the other side
+// of the same condition: when the cache serves every chunk, the flow must
+// continue (attach the cached vectors) even though the batch itself is empty.
+func TestEmbedChunksCacheHitOnlyBatchStillAttachesVectors(t *testing.T) {
+ comp, stub := withStubEmbedder(t, 4)
+ comp.param.Fields = []string{"text"}
+ store := newMemCacheStore()
+
+ warm := []schema.ChunkDoc{chunkWithID("c1", "alpha beta")}
+ if _, _, err := comp.embedChunks(context.Background(), "tenant", "kb", "doc.pdf", warm, store); err != nil {
+ t.Fatalf("warm-up embedChunks: %v", err)
+ }
+ callsAfterWarm := stub.calls.Load()
+ if callsAfterWarm == 0 {
+ t.Fatal("warm-up embedded nothing; the cache cannot be warm")
+ }
+
+ out, _, err := comp.embedChunks(context.Background(), "tenant", "kb", "doc.pdf", warm, store)
+ if err != nil {
+ t.Fatalf("cached embedChunks: %v", err)
+ }
+ // The title vector is not cached, so a fully cached pass re-encodes the
+ // title (the weighted merge needs it) but must not re-embed content.
+ if got, want := stub.calls.Load(), callsAfterWarm+1; got != want {
+ t.Fatalf("embedder calls = %d after the cached pass, want %d (title only)", got, want)
+ }
+ last := stub.callInputs[len(stub.callInputs)-1]
+ if len(last) != 1 || last[0] != "doc.pdf" {
+ t.Fatalf("cached pass re-embedded content: last call inputs = %v, want the title only", last)
+ }
+ if !hasEmbeddingVector(out[0]) {
+ t.Error("the cache-served chunk must still carry its vector")
+ }
+}
diff --git a/internal/ingestion/component/tokenizer_table_tag_test.go b/internal/ingestion/component/tokenizer_table_tag_test.go
new file mode 100644
index 0000000000..f9201c4dda
--- /dev/null
+++ b/internal/ingestion/component/tokenizer_table_tag_test.go
@@ -0,0 +1,110 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+//go:build integration
+
+package component
+
+import (
+ "context"
+ "testing"
+)
+
+// TestTokenizerComponent_Embedding_SkipsOnlyEmptyTableChunk: in a batch where
+// one chunk is nothing but an empty table, only the chunk with text may reach
+// the model. Python strips table-structure tags before the emptiness check
+// (rag/flow/tokenizer/tokenizer.py:86), so an empty-table chunk contributes no
+// embeddable text and the title encode is skipped too (no content to embed).
+func TestTokenizerComponent_Embedding_SkipsOnlyEmptyTableChunk(t *testing.T) {
+ requireTokenizerPool(t)
+ c, stub := withStubEmbedder(t, 2)
+ out, err := c.Invoke(context.Background(), nil, map[string]any{
+ "name": "doc.pdf",
+ "kb_id": "kb-1",
+ "output_format": "chunks",
+ "chunks": []map[string]any{
+ {"text": "<table><tr><td></td></tr></table>"},
+ {"text": "real text"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ // One title encode plus one content encode for the surviving chunk.
+ if got := stub.calls.Load(); got != 2 {
+ t.Fatalf("embedder calls = %d, want 2 (title + the one chunk with text)", got)
+ }
+ if len(stub.callInputs) != 2 || len(stub.callInputs[1]) != 1 || stub.callInputs[1][0] != "real text" {
+ t.Fatalf("content call inputs = %v, want [real text]", stub.callInputs)
+ }
+ chunks, _ := out["chunks"].([]map[string]any)
+ if len(chunks) != 2 {
+ t.Fatalf("output chunks = %d, want 2 (a skipped chunk stays in the payload)", len(chunks))
+ }
+ if _, ok := chunks[0]["q_2_vec"]; ok {
+ t.Error("the empty-table chunk must not carry a vector")
+ }
+ if _, ok := chunks[1]["q_2_vec"]; !ok {
+ t.Error("the chunk with text must carry a vector")
+ }
+}
+
+// TestTokenizerComponent_Embedding_TableWithTextIsEmbeddedStripped: what
+// reaches the model is the stripped text, not the raw HTML — Python appends
+// cleaned_txt to the embedding batch (tokenizer.py:89), so the tags are
+// replaced by spaces on the way in.
+func TestTokenizerComponent_Embedding_TableWithTextIsEmbeddedStripped(t *testing.T) {
+ requireTokenizerPool(t)
+ c, stub := withStubEmbedder(t, 2)
+ _, err := c.Invoke(context.Background(), nil, map[string]any{
+ "name": "doc.pdf",
+ "kb_id": "kb-1",
+ "output_format": "chunks",
+ "chunks": []map[string]any{
+ {"text": "<table><tr><td>hello</td></tr></table>"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ if len(stub.callInputs) != 2 || stub.callInputs[1][0] != "hello" {
+ t.Fatalf("content call inputs = %v, want the stripped text [hello]", stub.callInputs)
+ }
+}
+
+// TestTokenizerComponent_Embedding_LongAttributeTableTagIsNotStripped pins the
+// Python parity quirk: the attribute group in the strip pattern is bounded to
+// 12 non-tag characters, so a tag with longer attributes survives the strip —
+// only its attribute-free closing tag is replaced by a space.
+func TestTokenizerComponent_Embedding_LongAttributeTableTagIsNotStripped(t *testing.T) {
+ requireTokenizerPool(t)
+ c, stub := withStubEmbedder(t, 2)
+ _, err := c.Invoke(context.Background(), nil, map[string]any{
+ "name": "doc.pdf",
+ "kb_id": "kb-1",
+ "output_format": "chunks",
+ "chunks": []map[string]any{
+ {"text": `<table class="a-longer-class-name">cell</table>`},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ want := `<table class="a-longer-class-name">cell`
+ if len(stub.callInputs) != 2 || stub.callInputs[1][0] != want {
+ t.Fatalf("content call inputs = %v, want [%s]", stub.callInputs, want)
+ }
+}