RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(ingestion): port Python's table-tag strip to the tokenizer embedding path (#21156)
xugangqiangf2b9d65ef1ab
diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.goindex 9d6de83931..525eea0efb 100644--- a/internal/ingestion/component/tokenizer.go+++ b/internal/ingestion/component/tokenizer.go@@ -188,6 +188,23 @@ func resolveEmbedderMaxTokens(embedder Embedder) int {return embedder.MaxTokens()}+// tableStructureTagRe mirrors the Python tokenizer's cleaned_txt step+// (rag/flow/tokenizer/tokenizer.py:86): table-structure tags are replaced+// with a space before the emptiness check and before the text is embedded,+// so a chunk that is nothing but an empty table carries no embeddable+// content. The attribute group is deliberately bounded to 12 non-tag+// characters, exactly like the Python pattern: a tag with longer attributes+// survives the strip. Do not "fix" the bound — it is parity, not a bug.+var tableStructureTagRe = regexp.MustCompile(`</?(table|td|caption|tr|th)( [^<>]{0,12})?>`)++// cleanEmbedText applies the table-structure strip and trims, mirroring+// Python's cleaned_txt. The stripped text is both the emptiness check and+// what gets embedded — Python appends cleaned_txt to the embedding batch+// (tokenizer.py:89), not the raw chunk.+func cleanEmbedText(raw string) string {+ return strings.TrimSpace(tableStructureTagRe.ReplaceAllString(raw, " "))+}+// EmbedderResolver resolves the embedder and its dataset-bound embedding-model// id for one tokenizer invocation. The resolver derives the model exclusively// from the knowledgebase's configured embd_id (see internal/ingestion/task/@@ -470,7 +487,10 @@ func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, nacacheHits := 0for i, ck := range chunks {raw := concatFields(ck, c.param.Fields)- txt := strings.TrimSpace(raw)+ // Python parity (tokenizer.py:86-89): strip table-structure tags+ // before the emptiness check and embed the stripped text, so an+ // empty-table chunk contributes nothing to the batch.+ txt := cleanEmbedText(raw)if txt == "" {continue}@@ -498,11 +518,12 @@ func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, natruncs = append(truncs, trunc)pairs = append(pairs, i)}- if len(texts) == 0 {- // Nothing to embed — but cache hits may still need to be written through.- if store == nil {- return chunks, 0, nil- }+ if len(texts) == 0 && cacheHits == 0 {+ // Nothing to embed and nothing to serve from cache: return before the+ // title encode, exactly like Python's `if not texts: return` — an+ // all-skipped batch must not burn a title embedding. (When the cache+ // serves every chunk the flow continues so those vectors attach.)+ return chunks, 0, nil}trimmedName := strings.TrimSpace(name)@@ -1070,12 +1091,11 @@ func requiresFullTextTokens(ck schema.ChunkDoc) bool {return strings.TrimSpace(ck.Summary) != "" || strings.TrimSpace(schema.ContextualText(ck)) != ""}+// requiresEmbeddingVector uses the same cleaning as the embed path+// (cleanEmbedText): a chunk whose cleaned text is empty was skipped from the+// embedding batch and therefore legitimately carries no vector.func requiresEmbeddingVector(ck schema.ChunkDoc, fields []string) bool {- return strings.TrimSpace(cleanEmbeddingText(concatFields(ck, fields))) != ""-}--func cleanEmbeddingText(text string) string {- return strings.TrimSpace(text)+ return cleanEmbedText(concatFields(ck, fields)) != ""}func hasEmbeddingVector(ck schema.ChunkDoc) bool {diff --git a/internal/ingestion/component/tokenizer_clean_embed_text_test.go b/internal/ingestion/component/tokenizer_clean_embed_text_test.gonew file mode 100644index 0000000000..cea1038750--- /dev/null+++ b/internal/ingestion/component/tokenizer_clean_embed_text_test.go@@ -0,0 +1,154 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package component++import (+ "context"+ "strings"+ "testing"++ "ragflow/internal/ingestion/component/schema"+)++// TestCleanEmbedText pins the table-structure strip that mirrors Python's+// cleaned_txt (rag/flow/tokenizer/tokenizer.py:86). The contract is a direct+// regex port, quirks included: case-sensitive tag names, and an attribute+// group bounded to 12 non-tag characters.+func TestCleanEmbedText(t *testing.T) {+ cases := []struct {+ name string+ in string+ want string+ }{+ {"empty", "", ""},+ {"whitespace only", " \n\t ", ""},+ {"plain text unchanged", "just words", "just words"},+ {"text keeps inner whitespace", "a b\nc", "a b\nc"},++ {"empty table", "<table></table>", ""},+ {"empty table rows cells", "<table><tr><td></td></tr></table>", ""},+ {"whitespace-only table", "<table> </table>", ""},+ {"empty caption", "<caption></caption>", ""},+ {"empty th", "<th></th>", ""},++ {"table with text", "<table><tr><td>hello</td></tr></table>", "hello"},+ // Each replaced tag contributes one space; Python's re.sub does not+ // collapse runs of spaces, and neither does this port.+ {"text around table", "before<table><tr><td>x</td></tr></table>after", "before x after"},+ {"nested tables", "<table><tr><td><table><tr><td>deep</td></tr></table></td></tr></table>", "deep"},+ {"all tag kinds", "<table><caption>cap</caption><tr><th>h</th><td>d</td></tr></table>", "cap h d"},++ {"closing tags alone", "</table></td>", ""},+ {"attributes within bound", `<td colspan="2">x`, "x"},+ {"attribute bound exactly twelve", `<td class="abc">x`, "x"},+ {"attribute over bound survives", `<td class="abcde">x`, `<td class="abcde">x`},+ {"long attribute opening survives closing does not", `<table class="a-longer-class-name">cell</table>`, `<table class="a-longer-class-name">cell`},++ {"uppercase tags are not stripped (case-sensitive parity)", "<TABLE><TR><TD>x</TD></TR></TABLE>", "<TABLE><TR><TD>x</TD></TR></TABLE>"},+ {"mixed case tag not stripped", "<Table>x</Table>", "<Table>x</Table>"},++ {"unclosed tag untouched", "<table", "<table"},+ {"angle bracket soup untouched", "a < b > c", "a < b > c"},+ {"unicode around tags", "中文<table><tr><td>内容</td></tr></table>尾", "中文 内容 尾"},+ }+ for _, c := range cases {+ t.Run(c.name, func(t *testing.T) {+ if got := cleanEmbedText(c.in); got != c.want {+ t.Fatalf("cleanEmbedText(%q) = %q, want %q", c.in, got, c.want)+ }+ })+ }+}++// TestCleanEmbedTextStripsEveryMatch guards the global (not first-match)+// substitution: a long text with many table tags must have all of them+// replaced, not just the first.+func TestCleanEmbedTextStripsEveryMatch(t *testing.T) {+ in := "<td>a</td><td>b</td><td>c</td>"+ got := cleanEmbedText(in)+ if strings.Contains(got, "<td>") || strings.Contains(got, "</td>") {+ t.Fatalf("cleanEmbedText(%q) = %q, want every tag replaced", in, got)+ }+ if got != "a b c" {+ t.Fatalf("cleanEmbedText(%q) = %q, want %q", in, got, "a b c")+ }+}++// TestEmbedChunksAllSkippedWithCacheStoreSkipsTitleEncode: when every chunk's+// cleaned text is empty and no cache entry can serve them, nothing may be+// embedded — not even the title. Python returns before the title encode when+// the batch is empty (tokenizer.py:92-93); a configured cache store does not+// change that, it only matters when it actually serves a vector.+func TestEmbedChunksAllSkippedWithCacheStoreSkipsTitleEncode(t *testing.T) {+ comp, stub := withStubEmbedder(t, 4)+ comp.param.Fields = []string{"text"}+ chunks := []schema.ChunkDoc{+ chunkWithID("c1", "<table><tr><td></td></tr></table>"),+ chunkWithID("c2", " "),+ }++ out, consumed, err := comp.embedChunks(context.Background(), "tenant", "kb", "doc.pdf", chunks, newMemCacheStore())+ if err != nil {+ t.Fatalf("embedChunks: %v", err)+ }+ if got := stub.calls.Load(); got != 0 {+ t.Fatalf("embedder calls = %d, want 0 (an all-skipped batch embeds nothing, title included)", got)+ }+ if consumed != 0 {+ t.Fatalf("embedding_token_consumption = %d, want 0", consumed)+ }+ for i, ck := range out {+ if hasEmbeddingVector(ck) {+ t.Errorf("chunk[%d] unexpectedly carries a vector", i)+ }+ }+}++// TestEmbedChunksCacheHitOnlyBatchStillAttachesVectors guards the other side+// of the same condition: when the cache serves every chunk, the flow must+// continue (attach the cached vectors) even though the batch itself is empty.+func TestEmbedChunksCacheHitOnlyBatchStillAttachesVectors(t *testing.T) {+ comp, stub := withStubEmbedder(t, 4)+ comp.param.Fields = []string{"text"}+ store := newMemCacheStore()++ warm := []schema.ChunkDoc{chunkWithID("c1", "alpha beta")}+ if _, _, err := comp.embedChunks(context.Background(), "tenant", "kb", "doc.pdf", warm, store); err != nil {+ t.Fatalf("warm-up embedChunks: %v", err)+ }+ callsAfterWarm := stub.calls.Load()+ if callsAfterWarm == 0 {+ t.Fatal("warm-up embedded nothing; the cache cannot be warm")+ }++ out, _, err := comp.embedChunks(context.Background(), "tenant", "kb", "doc.pdf", warm, store)+ if err != nil {+ t.Fatalf("cached embedChunks: %v", err)+ }+ // The title vector is not cached, so a fully cached pass re-encodes the+ // title (the weighted merge needs it) but must not re-embed content.+ if got, want := stub.calls.Load(), callsAfterWarm+1; got != want {+ t.Fatalf("embedder calls = %d after the cached pass, want %d (title only)", got, want)+ }+ last := stub.callInputs[len(stub.callInputs)-1]+ if len(last) != 1 || last[0] != "doc.pdf" {+ t.Fatalf("cached pass re-embedded content: last call inputs = %v, want the title only", last)+ }+ if !hasEmbeddingVector(out[0]) {+ t.Error("the cache-served chunk must still carry its vector")+ }+}diff --git a/internal/ingestion/component/tokenizer_table_tag_test.go b/internal/ingestion/component/tokenizer_table_tag_test.gonew file mode 100644index 0000000000..f9201c4dda--- /dev/null+++ b/internal/ingestion/component/tokenizer_table_tag_test.go@@ -0,0 +1,110 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++//go:build integration++package component++import (+ "context"+ "testing"+)++// TestTokenizerComponent_Embedding_SkipsOnlyEmptyTableChunk: in a batch where+// one chunk is nothing but an empty table, only the chunk with text may reach+// the model. Python strips table-structure tags before the emptiness check+// (rag/flow/tokenizer/tokenizer.py:86), so an empty-table chunk contributes no+// embeddable text and the title encode is skipped too (no content to embed).+func TestTokenizerComponent_Embedding_SkipsOnlyEmptyTableChunk(t *testing.T) {+ requireTokenizerPool(t)+ c, stub := withStubEmbedder(t, 2)+ out, err := c.Invoke(context.Background(), nil, map[string]any{+ "name": "doc.pdf",+ "kb_id": "kb-1",+ "output_format": "chunks",+ "chunks": []map[string]any{+ {"text": "<table><tr><td></td></tr></table>"},+ {"text": "real text"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ // One title encode plus one content encode for the surviving chunk.+ if got := stub.calls.Load(); got != 2 {+ t.Fatalf("embedder calls = %d, want 2 (title + the one chunk with text)", got)+ }+ if len(stub.callInputs) != 2 || len(stub.callInputs[1]) != 1 || stub.callInputs[1][0] != "real text" {+ t.Fatalf("content call inputs = %v, want [real text]", stub.callInputs)+ }+ chunks, _ := out["chunks"].([]map[string]any)+ if len(chunks) != 2 {+ t.Fatalf("output chunks = %d, want 2 (a skipped chunk stays in the payload)", len(chunks))+ }+ if _, ok := chunks[0]["q_2_vec"]; ok {+ t.Error("the empty-table chunk must not carry a vector")+ }+ if _, ok := chunks[1]["q_2_vec"]; !ok {+ t.Error("the chunk with text must carry a vector")+ }+}++// TestTokenizerComponent_Embedding_TableWithTextIsEmbeddedStripped: what+// reaches the model is the stripped text, not the raw HTML — Python appends+// cleaned_txt to the embedding batch (tokenizer.py:89), so the tags are+// replaced by spaces on the way in.+func TestTokenizerComponent_Embedding_TableWithTextIsEmbeddedStripped(t *testing.T) {+ requireTokenizerPool(t)+ c, stub := withStubEmbedder(t, 2)+ _, err := c.Invoke(context.Background(), nil, map[string]any{+ "name": "doc.pdf",+ "kb_id": "kb-1",+ "output_format": "chunks",+ "chunks": []map[string]any{+ {"text": "<table><tr><td>hello</td></tr></table>"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ if len(stub.callInputs) != 2 || stub.callInputs[1][0] != "hello" {+ t.Fatalf("content call inputs = %v, want the stripped text [hello]", stub.callInputs)+ }+}++// TestTokenizerComponent_Embedding_LongAttributeTableTagIsNotStripped pins the+// Python parity quirk: the attribute group in the strip pattern is bounded to+// 12 non-tag characters, so a tag with longer attributes survives the strip —+// only its attribute-free closing tag is replaced by a space.+func TestTokenizerComponent_Embedding_LongAttributeTableTagIsNotStripped(t *testing.T) {+ requireTokenizerPool(t)+ c, stub := withStubEmbedder(t, 2)+ _, err := c.Invoke(context.Background(), nil, map[string]any{+ "name": "doc.pdf",+ "kb_id": "kb-1",+ "output_format": "chunks",+ "chunks": []map[string]any{+ {"text": `<table class="a-longer-class-name">cell</table>`},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ want := `<table class="a-longer-class-name">cell`+ if len(stub.callInputs) != 2 || stub.callInputs[1][0] != want {+ t.Fatalf("content call inputs = %v, want [%s]", stub.callInputs, want)+ }+}