RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(tokenizer): never send an empty input to the embedding provider (#21439)
1BugKiller2a5c4444ffd2
diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.goindex 525eea0efb..fc7f0eaf8b 100644--- a/internal/ingestion/component/tokenizer.go+++ b/internal/ingestion/component/tokenizer.go@@ -485,6 +485,14 @@ func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, na// cacheHits counts the chunks served from the per-chunk embedding cache;// they are already-resolved work, so the progress fraction starts from them.cacheHits := 0+ // fieldsResolvedNoText counts chunks where the configured fields produced+ // nothing at all, which is the signature of a `fields` value naming a chunk+ // field this pipeline does not produce.+ fieldsResolvedNoText := 0+ // emptyAfterTrim counts chunks whose text survived that check but produced+ // nothing after trimming, which means the embedder's trimmer returned no+ // text for non-empty input.+ emptyAfterTrim := 0for i, ck := range chunks {raw := concatFields(ck, c.param.Fields)// Python parity (tokenizer.py:86-89): strip table-structure tags@@ -492,6 +500,15 @@ func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, na// empty-table chunk contributes nothing to the batch.txt := cleanEmbedText(raw)if txt == "" {+ // Two different causes reach this branch and only one is a defect:+ // no configured field produced text at all (misconfiguration),+ // versus a chunk that really is only table tags or whitespace (the+ // supported empty-table case, which must stay silent). Only a+ // literally empty concatFields result is the first — whitespace-only+ // content is still content a field produced.+ if raw == "" {+ fieldsResolvedNoText+++ }continue}// Per-chunk embedding cache: identical (dataset embd_id, chunk, embedded@@ -504,6 +521,19 @@ func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, na// every model would collapse onto one key and served vectors could come// from a different model.trunc := trimForEmbedding(embedder, txt)+ // A provider rejects the WHOLE batch when any input carries no content+ // token — vLLM answers 400 "Input validation error: `inputs` cannot be+ // empty" — so an empty input must never reach the batch. The+ // EmbedderTrimmer seam promises nothing about the result, so a trimmer+ // that returns no text for non-empty input is handled by embedding the+ // pre-trim text: that keeps this chunk's vector requirement intact+ // (validateTokenizerOutputs decides embeddability on the pre-trim text)+ // and lets the embedder's own over-limit ladder re-cut it if the+ // untrimmed text is too long.+ if strings.TrimSpace(trunc) == "" {+ emptyAfterTrim+++ trunc = txt+ }if chunkID, ok := ck.GetExtraString("id"); ok && embdID != "" && store != nil {if cached, hit := chunkcache.Get(ctx, store, chunkcache.Key("emb", embdID, chunkID, trunc)); hit {var vec []float64@@ -518,6 +548,20 @@ func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, natruncs = append(truncs, trunc)pairs = append(pairs, i)}+ // Both warnings sit outside the early return on purpose: a mixed batch can+ // embed some chunks and drop others, and the dropped ones must still be+ // reported. Each reports how many chunks it covers, not the batch size.+ if fieldsResolvedNoText > 0 {+ common.Warn("Tokenizer: configured fields produced no text for some chunks; they were skipped",+ zap.Strings("fields", c.param.Fields),+ zap.Int("chunks", fieldsResolvedNoText),+ zap.Int("total_chunks", len(chunks)))+ }+ if emptyAfterTrim > 0 {+ common.Warn("Tokenizer: embedder returned no text after trimming; embedded the untrimmed text instead",+ zap.Int("chunks", emptyAfterTrim),+ zap.Int("total_chunks", len(chunks)))+ }if len(texts) == 0 && cacheHits == 0 {// Nothing to embed and nothing to serve from cache: return before the// title encode, exactly like Python's `if not texts: return` — andiff --git a/internal/ingestion/component/tokenizer_embed_batch_guard_test.go b/internal/ingestion/component/tokenizer_embed_batch_guard_test.gonew file mode 100644index 0000000000..7c5cde41b5--- /dev/null+++ b/internal/ingestion/component/tokenizer_embed_batch_guard_test.go@@ -0,0 +1,184 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package component++import (+ "context"+ "strings"+ "testing"++ "ragflow/internal/ingestion/component/schema"+)++// emptyingTrimmer emulates an EmbedderTrimmer that returns no text. The seam is+// optional and its signature carries no non-empty guarantee, so this is a+// legitimate implementation; a provider rejects the whole batch when such an+// input reaches it (vLLM: 400 "`inputs` cannot be empty").+type emptyingTrimmer struct{ *stubEmbedder }++func (e emptyingTrimmer) Trim(string) (string, int) { return "", 0 }++// configurableTrimmer is the transparent counterpart: it implements the trimmer+// seam but returns the text unchanged, so trimForEmbedding never needs the+// cl100k BPE table these unit tests must not depend on.+type configurableTrimmer struct{ *stubEmbedder }++func (c configurableTrimmer) Trim(text string) (string, int) { return text, 0 }++// TestEmbedChunksNeverSendsEmptyInput pins that the embedding batch never+// carries an empty string, whatever the trimmer returns. Sending one costs the+// entire document: the provider rejects the request as a whole.+func TestEmbedChunksNeverSendsEmptyInput(t *testing.T) {+ stub := newStubEmbedder(4)+ embdID := "embd-test"+ comp, err := NewTokenizerComponentWithResolver(nil, func(ctx context.Context, _, _ string) (Embedder, string, error) {+ return emptyingTrimmer{stub}, embdID, nil+ })+ if err != nil {+ t.Fatalf("NewTokenizerComponentWithResolver: %v", err)+ }+ tc := comp.(*TokenizerComponent)+ tc.param.Fields = []string{"text"}+ chunks := []schema.ChunkDoc{chunkWithID("c1", "text that cannot be trimmed to nothing")}++ if _, _, err := tc.embedChunks(context.Background(), "tenant", "kb", "doc.md", chunks, newMemCacheStore()); err != nil {+ t.Fatalf("embedChunks: %v", err)+ }+ // The chunk has text, so a content call must have happened (the title call+ // comes first). Without this the loop below would pass vacuously if some+ // future change skipped the chunk entirely.+ if len(stub.callInputs) < 2 {+ t.Fatalf("expected a title call and a content call, got %v", stub.callInputs)+ }+ for callIdx, in := range stub.callInputs {+ for _, s := range in {+ if strings.TrimSpace(s) == "" {+ t.Fatalf("call %d embedded an empty input %q; the provider rejects the whole batch (all inputs per call: %v)",+ callIdx, s, stub.callInputs)+ }+ }+ }+}++// TestEmbedChunksKeepsVectorRequirementWhenTrimIsEmpty covers the consistency+// CodeRabbit raised on #21439: validateTokenizerOutputs decides embeddability+// on the pre-trim text, so a chunk that has text must still end up with a+// vector even when the trimmer returns nothing. Otherwise the guard would trade+// the provider's 400 for "missing embedding vector".+func TestEmbedChunksKeepsVectorRequirementWhenTrimIsEmpty(t *testing.T) {+ stub := newStubEmbedder(4)+ embdID := "embd-test"+ comp, err := NewTokenizerComponentWithResolver(nil, func(ctx context.Context, _, _ string) (Embedder, string, error) {+ return emptyingTrimmer{stub}, embdID, nil+ })+ if err != nil {+ t.Fatalf("NewTokenizerComponentWithResolver: %v", err)+ }+ tc := comp.(*TokenizerComponent)+ tc.param.Fields = []string{"text"}+ chunks := []schema.ChunkDoc{chunkWithID("c1", "text the pipeline produced")}++ out, _, err := tc.embedChunks(context.Background(), "tenant", "kb", "doc.md", chunks, newMemCacheStore())+ if err != nil {+ t.Fatalf("embedChunks: %v", err)+ }+ if !hasEmbeddingVector(out[0]) {+ t.Fatal("chunk with pre-trim text carries no vector; validateTokenizerOutputs would fail with 'missing embedding vector'")+ }+ if err := validateTokenizerOutputs(out, []string{"embedding"}, tc.param.Fields, "kb"); err != nil {+ t.Fatalf("validateTokenizerOutputs rejected the output: %v", err)+ }+ // Negative control: the assertion above only means something if validation+ // actually demands a vector for this configuration. A chunk with the same+ // text and no vector must be rejected.+ if err := validateTokenizerOutputs([]schema.ChunkDoc{chunkWithID("c9", "text the pipeline produced")}, []string{"embedding"}, tc.param.Fields, "kb"); err == nil {+ t.Fatal("validateTokenizerOutputs accepted a vector-less chunk; the positive assertion above would be vacuous")+ }+}++// TestEmbedChunksMixedBatchKeepsResolvableChunks covers the case the report's+// diagnostic must not hide: some chunks resolve to text and others do not. The+// resolvable ones must still be embedded and keep their vectors, and the whole+// document must not fail because of the others.+func TestEmbedChunksMixedBatchKeepsResolvableChunks(t *testing.T) {+ stub := newStubEmbedder(4)+ embdID := "embd-test"+ comp, err := NewTokenizerComponentWithResolver(nil, func(ctx context.Context, _, _ string) (Embedder, string, error) {+ return configurableTrimmer{stub}, embdID, nil+ })+ if err != nil {+ t.Fatalf("NewTokenizerComponentWithResolver: %v", err)+ }+ tc := comp.(*TokenizerComponent)+ tc.param.Fields = []string{"text"}+ chunks := []schema.ChunkDoc{+ chunkWithID("c1", "resolvable content"),+ chunkWithID("c2", "<table><tr><td></td></tr></table>"), // empty table: legitimately skipped+ }++ out, _, err := tc.embedChunks(context.Background(), "tenant", "kb", "doc.md", chunks, newMemCacheStore())+ if err != nil {+ t.Fatalf("embedChunks: %v", err)+ }+ if !hasEmbeddingVector(out[0]) {+ t.Error("chunk[0] has text but no vector; a mixed batch must still embed the resolvable chunks")+ }+ if hasEmbeddingVector(out[1]) {+ t.Error("chunk[1] is an empty table and must stay vector-less")+ }+ for _, in := range stub.callInputs {+ for _, s := range in {+ if strings.TrimSpace(s) == "" {+ t.Fatalf("embedded an empty input %q; provider rejects the whole batch", s)+ }+ }+ }+}++// TestEmbedChunksSkipsEveryChunkWhenFieldsResolveToNothing covers the+// configuration that produced the report: `fields` names a chunk field the+// pipeline does not fill, so every chunk resolves to nothing. The component must+// not call the provider at all and the caller keeps its chunks; the warning it+// emits names the configured fields (asserted only indirectly here — this test+// pins the behaviour, not the log line).+func TestEmbedChunksSkipsEveryChunkWhenFieldsResolveToNothing(t *testing.T) {+ comp, stub := withStubEmbedder(t, 4)+ comp.param.Fields = []string{"no_such_field"}+ chunks := []schema.ChunkDoc{+ chunkWithID("c1", "content the pipeline actually produced"),+ chunkWithID("c2", "more content"),+ }++ out, consumed, err := comp.embedChunks(context.Background(), "tenant", "kb", "doc.md", chunks, newMemCacheStore())+ if err != nil {+ t.Fatalf("embedChunks: %v", err)+ }+ if got := stub.calls.Load(); got != 0 {+ t.Fatalf("embedder calls = %d, want 0 when no configured field resolves to text (inputs: %v)", got, stub.callInputs)+ }+ if consumed != 0 {+ t.Fatalf("embedding_token_consumption = %d, want 0", consumed)+ }+ if len(out) != len(chunks) {+ t.Fatalf("returned %d chunks, want %d preserved", len(out), len(chunks))+ }+ for i, ck := range out {+ if hasEmbeddingVector(ck) {+ t.Errorf("chunk[%d] unexpectedly carries a vector", i)+ }+ }+}