Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(task_executor): cap extracted important_kwd terms to ES keyword limit (#17512)

SYED ALI ABBAS RAHIL7cb4e30e1915
diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.go
index 1b09d1ec2a..d6bbed6eee 100644
--- a/internal/ingestion/component/tokenizer.go
+++ b/internal/ingestion/component/tokenizer.go
@@ -85,6 +85,8 @@ import (
"regexp"
"slices"
"strings"
+ "unicode"
+ "unicode/utf8"
"go.uber.org/zap"
"gorm.io/gorm"
@@ -111,6 +113,34 @@ var htmlTableRE = regexp.MustCompile(`</?(table|td|caption|tr|th)( [^<>]{0,12})?
// EmbeddingResult carries a vector plus the model-reported token usage
// for that input batch entry.
+// esKeywordMaxTermBytes is the upper bound for a single term stored in an
+// Elasticsearch keyword field.
+const esKeywordMaxTermBytes = 32766
+
+// sanitizeKeywordTerm returns a keyword that fits into an Elasticsearch keyword
+// field. Small terms are returned unchanged; oversized terms are truncated at a
+// UTF-8 character boundary and trailing whitespace is stripped.
+func sanitizeKeywordTerm(term string) string {
+ term = strings.TrimSpace(term)
+ if term == "" {
+ return ""
+ }
+ if len(term) <= esKeywordMaxTermBytes {
+ return term
+ }
+ length := 0
+ end := 0
+ for _, r := range term {
+ size := utf8.RuneLen(r)
+ if length+size > esKeywordMaxTermBytes {
+ break
+ }
+ length += size
+ end += size
+ }
+ return strings.TrimRightFunc(term[:end], unicode.IsSpace)
+}
+
type EmbeddingResult struct {
Vector []float64
TokenCount int
@@ -701,7 +731,16 @@ func tokenizeChunks(chunks []schema.ChunkDoc, titleStem string, language string)
// semicolons and newlines stay part of the keyword rather than
// acting as separators. strings.Split also preserves empty
// elements, matching Python's "a,,b".split(",") == ["a","","b"].
- if err = ck.SetExtraValue("important_kwd", strings.Split(kw, ",")); err != nil {
+ // Each piece is then bounded to the ES keyword field byte limit and
+ // truncated at a UTF-8 character boundary.
+ rawParts := strings.Split(kw, ",")
+ kwdParts := make([]string, len(rawParts))
+ for i, part := range rawParts {
+ // Empty parts are intentionally preserved to match the contract
+ // tested by TestTokenizerComponent_ImportantKwd_PreservesEmptyElements.
+ kwdParts[i] = sanitizeKeywordTerm(part)
+ }
+ if err = ck.SetExtraValue("important_kwd", kwdParts); err != nil {
return fmt.Errorf("tokenizer: keyword list marshal: %w", err)
}
it, err := tok.Tokenize(kw)
diff --git a/internal/ingestion/component/tokenizer_unit_test.go b/internal/ingestion/component/tokenizer_unit_test.go
index e2fc00baf6..0372fb977f 100644
--- a/internal/ingestion/component/tokenizer_unit_test.go
+++ b/internal/ingestion/component/tokenizer_unit_test.go
@@ -787,6 +787,103 @@ func TestTokenizerComponent_ImportantKwd_PreservesEmptyElements(t *testing.T) {
}
}
+// TestSanitizeKeywordTerm verifies the ES keyword-field guard used by the Go
+// tokenizer path. It mirrors the Python regression tests in
+// tests/test_main.py::TestSanitizeKeywordTerm.
+func TestSanitizeKeywordTerm(t *testing.T) {
+ tests := []struct {
+ name string
+ term string
+ want string
+ wantByte int
+ }{
+ {
+ name: "small term unchanged",
+ term: "hello",
+ want: "hello",
+ },
+ {
+ name: "trims whitespace",
+ term: " world ",
+ want: "world",
+ },
+ {
+ name: "empty string",
+ term: "",
+ want: "",
+ },
+ {
+ name: "oversized single word truncated to byte limit",
+ term: strings.Repeat("x", 40000),
+ want: strings.Repeat("x", esKeywordMaxTermBytes),
+ wantByte: esKeywordMaxTermBytes,
+ },
+ {
+ name: "multi-byte characters truncated at rune boundary",
+ term: strings.Repeat("中", 20000),
+ want: strings.Repeat("中", esKeywordMaxTermBytes/3),
+ wantByte: esKeywordMaxTermBytes,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := sanitizeKeywordTerm(tt.term)
+ if got != tt.want {
+ t.Errorf("sanitizeKeywordTerm(%q) = %q, want %q", tt.term, got, tt.want)
+ }
+ if tt.wantByte > 0 && len(got) != tt.wantByte {
+ t.Errorf("len(sanitizeKeywordTerm(%q)) = %d, want %d", tt.term, len(got), tt.wantByte)
+ }
+ })
+ }
+}
+
+// TestTokenizerComponent_ImportantKwd_Bounded verifies that oversized keywords
+// are truncated to the ES keyword-field limit at the component level, while
+// normal comma-splitting behavior is preserved.
+func TestTokenizerComponent_ImportantKwd_Bounded(t *testing.T) {
+ tokenizer.SetEngineType("infinity")
+ defer tokenizer.SetEngineType("")
+
+ c, err := NewTokenizerComponent(map[string]any{
+ "search_method": []any{"full_text"},
+ })
+ if err != nil {
+ t.Fatalf("NewTokenizerComponent: %v", err)
+ }
+
+ oversized := strings.Repeat("x", 40000)
+ out, err := c.Invoke(context.Background(), nil, map[string]any{
+ "output_format": "chunks",
+ "chunks": []map[string]any{
+ {"text": "doc body", "keywords": "normal," + oversized + ",中文字"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ got, ok := out["chunks"].([]map[string]any)
+ if !ok || len(got) != 1 {
+ t.Fatalf("chunks = %v, want 1 chunk", out["chunks"])
+ }
+ kwd, ok := got[0]["important_kwd"].([]string)
+ if !ok {
+ t.Fatalf("important_kwd should be []string, got %T", got[0]["important_kwd"])
+ }
+ if len(kwd) != 3 {
+ t.Fatalf("important_kwd = %v, want 3 entries", kwd)
+ }
+ if kwd[0] != "normal" {
+ t.Errorf("important_kwd[0] = %q, want \"normal\"", kwd[0])
+ }
+ if len(kwd[1]) != esKeywordMaxTermBytes {
+ t.Errorf("important_kwd[1] byte length = %d, want %d", len(kwd[1]), esKeywordMaxTermBytes)
+ }
+ if kwd[2] != "中文字" {
+ t.Errorf("important_kwd[2] = %q, want \"中文字\"", kwd[2])
+ }
+}
+
// TestTextPayloadToChunks_B1B2 pins the text-payload adapter used for
// markdown/text/html payloads (tokenizer.go:578). This is the Go-correct
// counterpart to a Python-DSL bug where turning full_text off dropped the
diff --git a/rag/svr/task_executor_refactor/chunk_post_processor.py b/rag/svr/task_executor_refactor/chunk_post_processor.py
index ff2dc6b2a2..d92eb0d13f 100644
--- a/rag/svr/task_executor_refactor/chunk_post_processor.py
+++ b/rag/svr/task_executor_refactor/chunk_post_processor.py
@@ -43,6 +43,48 @@ from rag.prompts.generator import content_tagging, gen_metadata, keyword_extract
from rag.svr.task_executor_refactor.task_context import TaskContext
+# Elasticsearch keyword fields reject terms whose UTF-8 encoding exceeds
+# 32766 bytes. Split oversized terms so ingestion never fails because a
+# malformed LLM response produced a single huge "keyword".
+_ES_KEYWORD_MAX_TERM_BYTES = 32766
+
+
+def _sanitize_keyword_term(term: str) -> list[str]:
+ """Return keyword pieces that fit into an Elasticsearch keyword field.
+
+ If ``term`` is small enough it is returned as-is. Otherwise it is
+ truncated at a character boundary so the UTF-8 encoding never exceeds
+ the ES keyword limit. This avoids corrupting multi-byte characters by
+ slicing raw bytes.
+ """
+ term = term.strip()
+ if not term:
+ return []
+ term_byte_length = len(term.encode("utf-8"))
+ if term_byte_length <= _ES_KEYWORD_MAX_TERM_BYTES:
+ return [term]
+
+ logging.warning(
+ "Sanitizing oversized keyword term (%d bytes, limit %d)",
+ term_byte_length,
+ _ES_KEYWORD_MAX_TERM_BYTES,
+ )
+ length = 0
+ end = 0
+ for index, character in enumerate(term):
+ character_bytes = len(character.encode("utf-8"))
+ if length + character_bytes > _ES_KEYWORD_MAX_TERM_BYTES:
+ end = index
+ break
+ length += character_bytes
+ else:
+ end = len(term)
+ truncated = term[:end].rstrip()
+ if not truncated:
+ return []
+ return [truncated]
+
+
async def extract_keywords(docs: list[dict], ctx: TaskContext) -> None:
"""Extract keywords for chunks.
@@ -67,7 +109,7 @@ async def extract_keywords(docs: list[dict], ctx: TaskContext) -> None:
cached = await keyword_extraction(chat_mdl, d["content_with_weight"], topn)
set_llm_cache(chat_mdl.llm_name, d["content_with_weight"], cached, "keywords", {"topn": topn})
if cached:
- d["important_kwd"] = [k for k in re.split(r"[,,;;、\r\n]+", cached) if k.strip()]
+ d["important_kwd"] = [kw for k in re.split(r"[,,;;、\r\n]+", cached) for kw in _sanitize_keyword_term(k)]
d["important_tks"] = rag_tokenizer.tokenize(" ".join(d["important_kwd"]))
return
diff --git a/test/unit_test/rag/svr/task_executor_refactor/test_chunk_post_processor.py b/test/unit_test/rag/svr/task_executor_refactor/test_chunk_post_processor.py
new file mode 100644
index 0000000000..b25287aa43
--- /dev/null
+++ b/test/unit_test/rag/svr/task_executor_refactor/test_chunk_post_processor.py
@@ -0,0 +1,142 @@
+#
+# Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unit tests for chunk_post_processor keyword sanitization."""
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from rag.svr.task_executor_refactor.chunk_post_processor import (
+ _ES_KEYWORD_MAX_TERM_BYTES,
+ _sanitize_keyword_term,
+ extract_keywords,
+)
+from test.unit_test.rag.svr.task_executor_refactor.conftest import make_task_context
+
+
+class TestSanitizeKeywordTerm:
+ """Tests for _sanitize_keyword_term."""
+
+ def test_returns_small_term_as_is(self):
+ assert _sanitize_keyword_term("hello") == ["hello"]
+
+ def test_trims_whitespace(self):
+ assert _sanitize_keyword_term(" world ") == ["world"]
+
+ def test_drops_empty_term(self):
+ assert _sanitize_keyword_term(" ") == []
+
+ def test_truncates_oversized_whitespace_term(self):
+ oversized = "word " * 20000
+ pieces = _sanitize_keyword_term(oversized)
+ assert len(pieces) == 1
+ assert len(pieces[0].encode("utf-8")) <= _ES_KEYWORD_MAX_TERM_BYTES
+ assert pieces[0].startswith("word")
+
+ def test_truncates_oversized_single_word(self):
+ oversized = "x" * 40000
+ pieces = _sanitize_keyword_term(oversized)
+ assert len(pieces) == 1
+ assert len(pieces[0].encode("utf-8")) == _ES_KEYWORD_MAX_TERM_BYTES
+
+ def test_preserves_multi_byte_characters(self):
+ oversized = "中" * 20000
+ pieces = _sanitize_keyword_term(oversized)
+ assert len(pieces) == 1
+ assert len(pieces[0].encode("utf-8")) == _ES_KEYWORD_MAX_TERM_BYTES
+
+ def test_truncation_stops_at_character_boundary(self):
+ # 32766 is divisible by 3, so forcing a boundary just before a
+ # multi-byte character proves we are not slicing raw bytes.
+ term = "a" * (_ES_KEYWORD_MAX_TERM_BYTES - 2) + "中"
+ pieces = _sanitize_keyword_term(term)
+ assert len(pieces) == 1
+ assert "中" not in pieces[0]
+ assert len(pieces[0].encode("utf-8")) < _ES_KEYWORD_MAX_TERM_BYTES
+
+
+class TestExtractKeywords:
+ """Tests for extract_keywords."""
+
+ @pytest.mark.asyncio
+ async def test_extract_keywords_from_cache(self):
+ ctx = make_task_context(parser_config={"auto_keywords": 2})
+ docs = [{"content_with_weight": "some content"}]
+
+ with (
+ patch(
+ "rag.svr.task_executor_refactor.chunk_post_processor.resolve_model_config",
+ return_value=MagicMock(),
+ ),
+ patch("rag.svr.task_executor_refactor.chunk_post_processor.LLMBundle") as MockBundle,
+ patch(
+ "rag.svr.task_executor_refactor.chunk_post_processor.get_llm_cache",
+ return_value="key1, key2",
+ ),
+ patch(
+ "rag.svr.task_executor_refactor.chunk_post_processor.set_llm_cache",
+ ),
+ patch(
+ "rag.svr.task_executor_refactor.chunk_post_processor.rag_tokenizer.tokenize",
+ return_value="tokenized",
+ ) as mock_tokenize,
+ ):
+ chat_model = MagicMock()
+ chat_model.llm_name = "mock"
+ chat_model.__enter__ = MagicMock(return_value=chat_model)
+ chat_model.__exit__ = MagicMock(return_value=False)
+ MockBundle.return_value = chat_model
+
+ await extract_keywords(docs, ctx)
+
+ assert docs[0]["important_kwd"] == ["key1", "key2"]
+ assert docs[0]["important_tks"] == "tokenized"
+ mock_tokenize.assert_called_once_with("key1 key2")
+
+ @pytest.mark.asyncio
+ async def test_extract_keywords_avoids_oversized_terms(self):
+ ctx = make_task_context(parser_config={"auto_keywords": 2})
+ huge = "x" * 40000
+ docs = [{"content_with_weight": "some content"}]
+
+ with (
+ patch(
+ "rag.svr.task_executor_refactor.chunk_post_processor.resolve_model_config",
+ return_value=MagicMock(),
+ ),
+ patch("rag.svr.task_executor_refactor.chunk_post_processor.LLMBundle") as MockBundle,
+ patch(
+ "rag.svr.task_executor_refactor.chunk_post_processor.get_llm_cache",
+ return_value=huge,
+ ),
+ patch(
+ "rag.svr.task_executor_refactor.chunk_post_processor.set_llm_cache",
+ ),
+ patch(
+ "rag.svr.task_executor_refactor.chunk_post_processor.rag_tokenizer.tokenize",
+ ) as mock_tokenize,
+ ):
+ chat_model = MagicMock()
+ chat_model.llm_name = "mock"
+ chat_model.__enter__ = MagicMock(return_value=chat_model)
+ chat_model.__exit__ = MagicMock(return_value=False)
+ MockBundle.return_value = chat_model
+
+ await extract_keywords(docs, ctx)
+
+ assert len(docs[0]["important_kwd"]) == 1
+ assert len(docs[0]["important_kwd"][0].encode("utf-8")) == _ES_KEYWORD_MAX_TERM_BYTES
+ mock_tokenize.assert_called_once()