RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(task_executor): cap extracted important_kwd terms to ES keyword limit (#17512)
SYED ALI ABBAS RAHIL7cb4e30e1915
diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.goindex 1b09d1ec2a..d6bbed6eee 100644--- a/internal/ingestion/component/tokenizer.go+++ b/internal/ingestion/component/tokenizer.go@@ -85,6 +85,8 @@ import ("regexp""slices""strings"+ "unicode"+ "unicode/utf8""go.uber.org/zap""gorm.io/gorm"@@ -111,6 +113,34 @@ var htmlTableRE = regexp.MustCompile(`</?(table|td|caption|tr|th)( [^<>]{0,12})?// EmbeddingResult carries a vector plus the model-reported token usage// for that input batch entry.+// esKeywordMaxTermBytes is the upper bound for a single term stored in an+// Elasticsearch keyword field.+const esKeywordMaxTermBytes = 32766++// sanitizeKeywordTerm returns a keyword that fits into an Elasticsearch keyword+// field. Small terms are returned unchanged; oversized terms are truncated at a+// UTF-8 character boundary and trailing whitespace is stripped.+func sanitizeKeywordTerm(term string) string {+ term = strings.TrimSpace(term)+ if term == "" {+ return ""+ }+ if len(term) <= esKeywordMaxTermBytes {+ return term+ }+ length := 0+ end := 0+ for _, r := range term {+ size := utf8.RuneLen(r)+ if length+size > esKeywordMaxTermBytes {+ break+ }+ length += size+ end += size+ }+ return strings.TrimRightFunc(term[:end], unicode.IsSpace)+}+type EmbeddingResult struct {Vector []float64TokenCount int@@ -701,7 +731,16 @@ func tokenizeChunks(chunks []schema.ChunkDoc, titleStem string, language string)// semicolons and newlines stay part of the keyword rather than// acting as separators. strings.Split also preserves empty// elements, matching Python's "a,,b".split(",") == ["a","","b"].- if err = ck.SetExtraValue("important_kwd", strings.Split(kw, ",")); err != nil {+ // Each piece is then bounded to the ES keyword field byte limit and+ // truncated at a UTF-8 character boundary.+ rawParts := strings.Split(kw, ",")+ kwdParts := make([]string, len(rawParts))+ for i, part := range rawParts {+ // Empty parts are intentionally preserved to match the contract+ // tested by TestTokenizerComponent_ImportantKwd_PreservesEmptyElements.+ kwdParts[i] = sanitizeKeywordTerm(part)+ }+ if err = ck.SetExtraValue("important_kwd", kwdParts); err != nil {return fmt.Errorf("tokenizer: keyword list marshal: %w", err)}it, err := tok.Tokenize(kw)diff --git a/internal/ingestion/component/tokenizer_unit_test.go b/internal/ingestion/component/tokenizer_unit_test.goindex e2fc00baf6..0372fb977f 100644--- a/internal/ingestion/component/tokenizer_unit_test.go+++ b/internal/ingestion/component/tokenizer_unit_test.go@@ -787,6 +787,103 @@ func TestTokenizerComponent_ImportantKwd_PreservesEmptyElements(t *testing.T) {}}+// TestSanitizeKeywordTerm verifies the ES keyword-field guard used by the Go+// tokenizer path. It mirrors the Python regression tests in+// tests/test_main.py::TestSanitizeKeywordTerm.+func TestSanitizeKeywordTerm(t *testing.T) {+ tests := []struct {+ name string+ term string+ want string+ wantByte int+ }{+ {+ name: "small term unchanged",+ term: "hello",+ want: "hello",+ },+ {+ name: "trims whitespace",+ term: " world ",+ want: "world",+ },+ {+ name: "empty string",+ term: "",+ want: "",+ },+ {+ name: "oversized single word truncated to byte limit",+ term: strings.Repeat("x", 40000),+ want: strings.Repeat("x", esKeywordMaxTermBytes),+ wantByte: esKeywordMaxTermBytes,+ },+ {+ name: "multi-byte characters truncated at rune boundary",+ term: strings.Repeat("中", 20000),+ want: strings.Repeat("中", esKeywordMaxTermBytes/3),+ wantByte: esKeywordMaxTermBytes,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got := sanitizeKeywordTerm(tt.term)+ if got != tt.want {+ t.Errorf("sanitizeKeywordTerm(%q) = %q, want %q", tt.term, got, tt.want)+ }+ if tt.wantByte > 0 && len(got) != tt.wantByte {+ t.Errorf("len(sanitizeKeywordTerm(%q)) = %d, want %d", tt.term, len(got), tt.wantByte)+ }+ })+ }+}++// TestTokenizerComponent_ImportantKwd_Bounded verifies that oversized keywords+// are truncated to the ES keyword-field limit at the component level, while+// normal comma-splitting behavior is preserved.+func TestTokenizerComponent_ImportantKwd_Bounded(t *testing.T) {+ tokenizer.SetEngineType("infinity")+ defer tokenizer.SetEngineType("")++ c, err := NewTokenizerComponent(map[string]any{+ "search_method": []any{"full_text"},+ })+ if err != nil {+ t.Fatalf("NewTokenizerComponent: %v", err)+ }++ oversized := strings.Repeat("x", 40000)+ out, err := c.Invoke(context.Background(), nil, map[string]any{+ "output_format": "chunks",+ "chunks": []map[string]any{+ {"text": "doc body", "keywords": "normal," + oversized + ",中文字"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ got, ok := out["chunks"].([]map[string]any)+ if !ok || len(got) != 1 {+ t.Fatalf("chunks = %v, want 1 chunk", out["chunks"])+ }+ kwd, ok := got[0]["important_kwd"].([]string)+ if !ok {+ t.Fatalf("important_kwd should be []string, got %T", got[0]["important_kwd"])+ }+ if len(kwd) != 3 {+ t.Fatalf("important_kwd = %v, want 3 entries", kwd)+ }+ if kwd[0] != "normal" {+ t.Errorf("important_kwd[0] = %q, want \"normal\"", kwd[0])+ }+ if len(kwd[1]) != esKeywordMaxTermBytes {+ t.Errorf("important_kwd[1] byte length = %d, want %d", len(kwd[1]), esKeywordMaxTermBytes)+ }+ if kwd[2] != "中文字" {+ t.Errorf("important_kwd[2] = %q, want \"中文字\"", kwd[2])+ }+}+// TestTextPayloadToChunks_B1B2 pins the text-payload adapter used for// markdown/text/html payloads (tokenizer.go:578). This is the Go-correct// counterpart to a Python-DSL bug where turning full_text off dropped thediff --git a/rag/svr/task_executor_refactor/chunk_post_processor.py b/rag/svr/task_executor_refactor/chunk_post_processor.pyindex ff2dc6b2a2..d92eb0d13f 100644--- a/rag/svr/task_executor_refactor/chunk_post_processor.py+++ b/rag/svr/task_executor_refactor/chunk_post_processor.py@@ -43,6 +43,48 @@ from rag.prompts.generator import content_tagging, gen_metadata, keyword_extractfrom rag.svr.task_executor_refactor.task_context import TaskContext+# Elasticsearch keyword fields reject terms whose UTF-8 encoding exceeds+# 32766 bytes. Split oversized terms so ingestion never fails because a+# malformed LLM response produced a single huge "keyword".+_ES_KEYWORD_MAX_TERM_BYTES = 32766+++def _sanitize_keyword_term(term: str) -> list[str]:+ """Return keyword pieces that fit into an Elasticsearch keyword field.++ If ``term`` is small enough it is returned as-is. Otherwise it is+ truncated at a character boundary so the UTF-8 encoding never exceeds+ the ES keyword limit. This avoids corrupting multi-byte characters by+ slicing raw bytes.+ """+ term = term.strip()+ if not term:+ return []+ term_byte_length = len(term.encode("utf-8"))+ if term_byte_length <= _ES_KEYWORD_MAX_TERM_BYTES:+ return [term]++ logging.warning(+ "Sanitizing oversized keyword term (%d bytes, limit %d)",+ term_byte_length,+ _ES_KEYWORD_MAX_TERM_BYTES,+ )+ length = 0+ end = 0+ for index, character in enumerate(term):+ character_bytes = len(character.encode("utf-8"))+ if length + character_bytes > _ES_KEYWORD_MAX_TERM_BYTES:+ end = index+ break+ length += character_bytes+ else:+ end = len(term)+ truncated = term[:end].rstrip()+ if not truncated:+ return []+ return [truncated]++async def extract_keywords(docs: list[dict], ctx: TaskContext) -> None:"""Extract keywords for chunks.@@ -67,7 +109,7 @@ async def extract_keywords(docs: list[dict], ctx: TaskContext) -> None:cached = await keyword_extraction(chat_mdl, d["content_with_weight"], topn)set_llm_cache(chat_mdl.llm_name, d["content_with_weight"], cached, "keywords", {"topn": topn})if cached:- d["important_kwd"] = [k for k in re.split(r"[,,;;、\r\n]+", cached) if k.strip()]+ d["important_kwd"] = [kw for k in re.split(r"[,,;;、\r\n]+", cached) for kw in _sanitize_keyword_term(k)]d["important_tks"] = rag_tokenizer.tokenize(" ".join(d["important_kwd"]))returndiff --git a/test/unit_test/rag/svr/task_executor_refactor/test_chunk_post_processor.py b/test/unit_test/rag/svr/task_executor_refactor/test_chunk_post_processor.pynew file mode 100644index 0000000000..b25287aa43--- /dev/null+++ b/test/unit_test/rag/svr/task_executor_refactor/test_chunk_post_processor.py@@ -0,0 +1,142 @@+#+# Copyright 2024 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""Unit tests for chunk_post_processor keyword sanitization."""++from unittest.mock import MagicMock, patch++import pytest++from rag.svr.task_executor_refactor.chunk_post_processor import (+ _ES_KEYWORD_MAX_TERM_BYTES,+ _sanitize_keyword_term,+ extract_keywords,+)+from test.unit_test.rag.svr.task_executor_refactor.conftest import make_task_context+++class TestSanitizeKeywordTerm:+ """Tests for _sanitize_keyword_term."""++ def test_returns_small_term_as_is(self):+ assert _sanitize_keyword_term("hello") == ["hello"]++ def test_trims_whitespace(self):+ assert _sanitize_keyword_term(" world ") == ["world"]++ def test_drops_empty_term(self):+ assert _sanitize_keyword_term(" ") == []++ def test_truncates_oversized_whitespace_term(self):+ oversized = "word " * 20000+ pieces = _sanitize_keyword_term(oversized)+ assert len(pieces) == 1+ assert len(pieces[0].encode("utf-8")) <= _ES_KEYWORD_MAX_TERM_BYTES+ assert pieces[0].startswith("word")++ def test_truncates_oversized_single_word(self):+ oversized = "x" * 40000+ pieces = _sanitize_keyword_term(oversized)+ assert len(pieces) == 1+ assert len(pieces[0].encode("utf-8")) == _ES_KEYWORD_MAX_TERM_BYTES++ def test_preserves_multi_byte_characters(self):+ oversized = "中" * 20000+ pieces = _sanitize_keyword_term(oversized)+ assert len(pieces) == 1+ assert len(pieces[0].encode("utf-8")) == _ES_KEYWORD_MAX_TERM_BYTES++ def test_truncation_stops_at_character_boundary(self):+ # 32766 is divisible by 3, so forcing a boundary just before a+ # multi-byte character proves we are not slicing raw bytes.+ term = "a" * (_ES_KEYWORD_MAX_TERM_BYTES - 2) + "中"+ pieces = _sanitize_keyword_term(term)+ assert len(pieces) == 1+ assert "中" not in pieces[0]+ assert len(pieces[0].encode("utf-8")) < _ES_KEYWORD_MAX_TERM_BYTES+++class TestExtractKeywords:+ """Tests for extract_keywords."""++ @pytest.mark.asyncio+ async def test_extract_keywords_from_cache(self):+ ctx = make_task_context(parser_config={"auto_keywords": 2})+ docs = [{"content_with_weight": "some content"}]++ with (+ patch(+ "rag.svr.task_executor_refactor.chunk_post_processor.resolve_model_config",+ return_value=MagicMock(),+ ),+ patch("rag.svr.task_executor_refactor.chunk_post_processor.LLMBundle") as MockBundle,+ patch(+ "rag.svr.task_executor_refactor.chunk_post_processor.get_llm_cache",+ return_value="key1, key2",+ ),+ patch(+ "rag.svr.task_executor_refactor.chunk_post_processor.set_llm_cache",+ ),+ patch(+ "rag.svr.task_executor_refactor.chunk_post_processor.rag_tokenizer.tokenize",+ return_value="tokenized",+ ) as mock_tokenize,+ ):+ chat_model = MagicMock()+ chat_model.llm_name = "mock"+ chat_model.__enter__ = MagicMock(return_value=chat_model)+ chat_model.__exit__ = MagicMock(return_value=False)+ MockBundle.return_value = chat_model++ await extract_keywords(docs, ctx)++ assert docs[0]["important_kwd"] == ["key1", "key2"]+ assert docs[0]["important_tks"] == "tokenized"+ mock_tokenize.assert_called_once_with("key1 key2")++ @pytest.mark.asyncio+ async def test_extract_keywords_avoids_oversized_terms(self):+ ctx = make_task_context(parser_config={"auto_keywords": 2})+ huge = "x" * 40000+ docs = [{"content_with_weight": "some content"}]++ with (+ patch(+ "rag.svr.task_executor_refactor.chunk_post_processor.resolve_model_config",+ return_value=MagicMock(),+ ),+ patch("rag.svr.task_executor_refactor.chunk_post_processor.LLMBundle") as MockBundle,+ patch(+ "rag.svr.task_executor_refactor.chunk_post_processor.get_llm_cache",+ return_value=huge,+ ),+ patch(+ "rag.svr.task_executor_refactor.chunk_post_processor.set_llm_cache",+ ),+ patch(+ "rag.svr.task_executor_refactor.chunk_post_processor.rag_tokenizer.tokenize",+ ) as mock_tokenize,+ ):+ chat_model = MagicMock()+ chat_model.llm_name = "mock"+ chat_model.__enter__ = MagicMock(return_value=chat_model)+ chat_model.__exit__ = MagicMock(return_value=False)+ MockBundle.return_value = chat_model++ await extract_keywords(docs, ctx)++ assert len(docs[0]["important_kwd"]) == 1+ assert len(docs[0]["important_kwd"][0].encode("utf-8")) == _ES_KEYWORD_MAX_TERM_BYTES+ mock_tokenize.assert_called_once()